00000001:01000000:2.0F:1787760995.249586:0:2284:0:(lnet-crypto.c:359:cfs_crypto_performance_test()) Crypto hash algorithm adler32 speed = 1211 MB/s 00000001:01000000:2.0:1787760995.499886:0:2284:0:(lnet-crypto.c:359:cfs_crypto_performance_test()) Crypto hash algorithm crc32 speed = 1631 MB/s 00000001:01000000:2.0:1787760995.749659:0:2284:0:(lnet-crypto.c:359:cfs_crypto_performance_test()) Crypto hash algorithm crc32c speed = 10067 MB/s 00000020:02000000:1.0F:1787760996.053735:0:2313:0:(class_obd.c:771:obdclass_init()) Lustre: Build Version: 2.17.57_119_g5069d9c 00000400:02000000:1.0:1787760996.426333:0:2328:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.23@tcp [8/256/0/180] 00008000:02000000:2.0:1787760999.891263:0:2478:0:(echo_client.c:2574:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:01000004:2.0:1787761160.792254:0:5427:0:(obd_mount.c:923:lmd_print()) mount data: 00000020:01000004:2.0:1787761160.792259:0:5427:0:(obd_mount.c:925:lmd_print()) profile: lustre-client 00000020:01000004:2.0:1787761160.792260:0:5427:0:(obd_mount.c:926:lmd_print()) device: 192.168.203.123@tcp:/lustre 00000020:01000004:2.0:1787761160.792262:0:5427:0:(obd_mount.c:929:lmd_print()) options: user_xattr,flock 00000080:01200004:2.0:1787761160.792399:0:5427:0:(super25.c:131:lustre_fill_super()) VFS Op: sb 00000000a76153f5 00000080:01000004:2.0:1787761160.792407:0:5427:0:(super25.c:162:lustre_fill_super()) Mounting client lustre-client 00000020:01000004:2.0:1787761160.792422:0:5427:0:(obd_mount.c:411:lustre_start_mgc()) Start MGC 'MGC192.168.203.123@tcp' 00000020:01000004:2.0:1787761160.792467:0:5427:0:(obd_mount.c:230:lustre_start_simple()) Starting OBD MGC192.168.203.123@tcp (typ=mgc) 00000020:00000080:2.0:1787761160.792485:0:5427:0:(genops.c:413:class_newdev()) Allocate new device MGC192.168.203.123@tcp (00000000e2ea64c6) 00000020:00000080:2.0:1787761160.792497:0:5427:0:(obd_config.c:720:class_attach_name()) OBD: dev 0 attached type mgc with refcount 2 00000100:00100000:2.0:1787761160.792698:0:5427:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:2.0:1787761160.792720:0:5427:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:2.0:1787761160.792871:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 0 min 2 max 34 00000100:00100000:2.0:1787761160.792878:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_000' 00000100:00100000:3.0F:1787761160.794961:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 1 min 2 max 34 00000100:00100000:3.0:1787761160.794982:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_001' 00000100:00100000:3.0:1787761160.795693:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 0 min 2 max 34 00000100:00100000:3.0:1787761160.795701:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_000' 00000100:00100000:2.0:1787761160.796030:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 1 min 2 max 34 00000100:00100000:2.0:1787761160.796038:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_001' 00000100:00100000:3.0:1787761160.796670:0:5427:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:3.0:1787761160.797008:0:5427:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:3.0:1787761160.797192:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 0 min 3 max 34 00000100:00100000:3.0:1787761160.797197:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_000' 00000100:00100000:2.0:1787761160.800181:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 1 min 3 max 34 00000100:00100000:2.0:1787761160.800195:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_001' 00000100:00100000:1.0:1787761160.803351:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 2 min 3 max 34 00000100:00100000:1.0:1787761160.803364:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_002' 00000100:00100000:2.0:1787761160.803726:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 0 min 3 max 34 00000100:00100000:2.0:1787761160.803733:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_000' 00000100:00100000:3.0:1787761160.805324:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 1 min 3 max 34 00000100:00100000:3.0:1787761160.805337:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_001' 00000100:00100000:2.0:1787761160.810715:0:5427:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 2 min 3 max 34 00000100:00100000:2.0:1787761160.810733:0:5427:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_002' 00010000:00010000:3.0:1787761160.811696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761160.811700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761160.811702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761160.811983:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761160.811985:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761160.811987:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:2.0:1787761160.812623:0:5427:0:(ldlm_lib.c:102:import_set_conn()) imp 000000008075602e@MGC192.168.203.123@tcp: add connection 192.168.203.123@tcp at head 00010000:00010000:2.0:1787761160.812822:0:5427:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-MGC192.168.203.123@tcp-0 is initialized 00000040:01000000:2.0:1787761160.812830:0:5427:0:(llog_obd.c:197:llog_setup()) obd MGC192.168.203.123@tcp ctxt 1 is initialized 10000000:01000000:2.0:1787761160.816514:0:5442:0:(mgc_request.c:631:mgc_requeue_thread()) Starting requeue thread 00000020:00000080:0.0F:1787761160.816592:0:5427:0:(obd_config.c:826:class_setup()) finished setup of obd MGC192.168.203.123@tcp (uuid 94b87323-e84d-4c2a-846c-789bad072fad) 00000020:00000080:0.0:1787761160.816626:0:5427:0:(genops.c:1385:class_connect()) connect: client 94b87323-e84d-4c2a-846c-789bad072fad, cookie 0x88466b5949dc2e39 00000100:00080000:0.0:1787761160.816640:0:5427:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 0 00000100:00080000:0.0:1787761160.816653:0:5427:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import 000000008075602e using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761160.816691:0:5427:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000658785c3 x1874603274141824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761160.816718:0:5427:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 94b87323-e84d-4c2a-846c-789bad072fad->MGS 00000080:00200000:0.0:1787761160.816735:0:5427:0:(llite_lib.c:1509:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffff9ac4523c1000 (sb 00000000a76153f5) 00000080:01000000:0.0:1787761160.816802:0:5427:0:(llite_lib.c:1235:ll_options()) Parsing opts user_xattr,flock 00000080:00000004:0.0:1787761160.816803:0:5427:0:(llite_lib.c:1245:ll_options()) next opt=user_xattr 00000080:00000004:0.0:1787761160.816807:0:5427:0:(llite_lib.c:1245:ll_options()) next opt=flock 00000080:01000000:0.0:1787761160.816815:0:5427:0:(llite_lib.c:1543:ll_fill_super()) llite sb uuid: 522fbd36-e7cb-44ce-913c-7337604532e0 10000000:01000000:0.0:1787761160.817352:0:5427:0:(mgc_request.c:1944:mgc_process_config()) parse_log lustre-client from 0 10000000:01000000:0.0:1787761160.817356:0:5427:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffff9ac4523c1000 10000000:01000000:0.0:1787761160.817360:0:5427:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-sptlrpc-ffff9ac4601c9a48 10000000:01000000:0.0:1787761160.817363:0:5427:0:(mgc_request.c:68:mgc_name2resid()) log lustre-sptlrpc to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:0.0:1787761160.817370:0:5427:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff9ac4601c9a48 from 1 10000000:01000000:0.0:1787761160.817373:0:5427:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1787761160.817399:0:5427:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787761160.817409:0:5427:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787761160.817418:0:5427:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:0.0:1787761160.817432:0:5427:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@000000007ccb2c53 x1874603274141952/t0(0) o101->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761160.817440:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 0000000037ce192d going to sleep for 0 seconds 00010000:00010000:3.0:1787761160.818099:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761160.824504:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000658785c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274141824:192.168.203.123@tcp:250: 00000100:00100000:2.0:1787761160.831933:0:2344:0:(client.c:3125:ptlrpc_free_committed()) MGC192.168.203.123@tcp: committing for last_committed 0 gen 1 00000100:00080000:2.0:1787761160.831949:0:2344:0:(import.c:1132:ptlrpc_connect_interpret()) MGC192.168.203.123@tcp: connect to target with instance 0 00000100:00080000:2.0:1787761160.831959:0:2344:0:(import.c:956:ptlrpc_connect_set_flags()) MGC192.168.203.123@tcp: Resetting ns_connect_flags to server flags: 0xa000011001002060 10000000:01000000:2.0:1787761160.831964:0:2344:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:00080000:2.0:1787761160.831967:0:2344:0:(import.c:65:import_set_state_nolock()) 000000008075602e MGS: changing import state from CONNECTING to FULL 10000000:01000000:2.0:1787761160.831972:0:2344:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1787761160.831993:0:2344:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:00080000:2.0:1787761160.832003:0:2344:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 0000000037ce192d): req@000000007ccb2c53 x1874603274141952/t0(0) o101->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761160.832025:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000658785c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274141824:192.168.203.123@tcp:250: 00000100:00100000:1.0:1787761160.832735:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 0000000037ce192d going to sleep for 11 seconds 00000100:00100000:2.0:1787761160.849463:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000007ccb2c53 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274141952:192.168.203.123@tcp:101: 00010000:00010000:2.0:1787761160.849494:0:5427:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761160.849507:0:5427:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787761160.849606:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000000e51b809 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142080:192.168.203.123@tcp:501: 00000100:00100000:2.0:1787761160.849673:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 0000000031ab229d going to sleep for 11 seconds 00000100:00100000:3.0:1787761160.852245:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000000e51b809 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142080:192.168.203.123@tcp:501: 10000000:01000000:3.0:1787761160.852290:0:5427:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1787761160.852343:0:5427:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.852353:0:5427:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: 0000000043fac660/0x88466b5949dc2e40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787761160.852361:0:5427:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffff9ac4523c1000 10000000:01000000:3.0:1787761160.852367:0:5427:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 10000000:01000000:3.0:1787761160.852371:0:5427:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffff9ac4523c1000 10000000:01000000:3.0:1787761160.852373:0:5427:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:3.0:1787761160.852377:0:5427:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffff9ac4523c1000 10000000:01000000:3.0:1787761160.852379:0:5427:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 10000000:01000000:3.0:1787761160.852382:0:5427:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff9ac4523c1000 from 1 10000000:01000000:3.0:1787761160.852384:0:5427:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1787761160.852414:0:5427:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.852430:0:5427:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.852439:0:5427:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761160.852455:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142208:192.168.203.123@tcp:101: 00000100:00100000:3.0:1787761160.852558:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 0000000009769f4f going to sleep for 11 seconds 00000100:00100000:2.0:1787761160.855773:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142208:192.168.203.123@tcp:101: 00010000:00010000:2.0:1787761160.855813:0:5427:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761160.855825:0:5427:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787761160.855896:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000007c7e6d5c pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142336:192.168.203.123@tcp:501: 00000100:00100000:2.0:1787761160.855977:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000c69fc147 going to sleep for 11 seconds 00000100:00100000:2.0:1787761160.862244:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000007c7e6d5c pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142336:192.168.203.123@tcp:501: 00000100:00100000:2.0:1787761160.862316:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000007c7e6d5c pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142464:192.168.203.123@tcp:503: 00000100:00100000:2.0:1787761160.862426:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000c69fc147 going to sleep for 11 seconds 00000100:00100000:2.0:1787761160.866493:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000007c7e6d5c pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142464:192.168.203.123@tcp:503: 00000100:00100000:0.0:1787761160.866891:0:5446:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000daa728c8 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5446:1874603274142592:192.168.203.123@tcp:502: 00000100:00100000:0.0:1787761160.867003:0:5446:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d9ce4037 going to sleep for 11 seconds 00000100:00100000:1.0:1787761160.870204:0:5446:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000daa728c8 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5446:1874603274142592:192.168.203.123@tcp:502: 00000020:01000000:1.0:1787761160.870298:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1787761160.870309:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.870311:0:5446:0:(obd_config.c:1563:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:1.0:1787761160.870317:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.870320:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1787761160.870343:0:5446:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilov-ffff9ac4523c1000 (000000007ee6daba) 00000020:00000080:1.0:1787761160.870358:0:5446:0:(obd_config.c:720:class_attach_name()) OBD: dev 1 attached type lov with refcount 2 00000020:01000000:1.0:1787761160.870364:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.870366:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:1.0:1787761160.870845:0:5446:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilov-ffff9ac4523c1000 (uuid 522fbd36-e7cb-44ce-913c-7337604532e0) 00000020:01000000:1.0:1787761160.870852:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.870856:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.870858:0:5446:0:(obd_config.c:1563:class_process_config()) marker 4 (0x2) lustre-clilov lov setup 00000020:01000000:1.0:1787761160.870861:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1787761160.870864:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.870865:0:5446:0:(obd_config.c:1563:class_process_config()) marker 5 (0x1) lustre-clilmv lmv setup 00000020:01000000:1.0:1787761160.870868:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilmv-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.870869:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1787761160.870875:0:5446:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilmv-ffff9ac4523c1000 (00000000907f9af2) 00000020:00000080:1.0:1787761160.870884:0:5446:0:(obd_config.c:720:class_attach_name()) OBD: dev 2 attached type lmv with refcount 2 00000020:01000000:1.0:1787761160.870888:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilmv-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.870890:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:1.0:1787761160.872816:0:5446:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilmv-ffff9ac4523c1000 (uuid 522fbd36-e7cb-44ce-913c-7337604532e0) 00000020:01000000:1.0:1787761160.872833:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.872840:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.872842:0:5446:0:(obd_config.c:1563:class_process_config()) marker 5 (0x2) lustre-clilmv lmv setup 00000020:01000000:1.0:1787761160.872853:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1787761160.872858:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.872859:0:5446:0:(obd_config.c:1563:class_process_config()) marker 6 (0x1) lustre-client mount opts 00000020:00000080:1.0:1787761160.872863:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf007 00000020:00000080:1.0:1787761160.872864:0:5446:0:(obd_config.c:1516:class_process_config()) mountopt: profile lustre-client osc lustre-clilov mdc lustre-clilmv 00000020:01000000:1.0:1787761160.872867:0:5446:0:(obd_config.c:1149:class_add_profile()) Add profile lustre-client 00000020:01000000:1.0:1787761160.872872:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.872874:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.872875:0:5446:0:(obd_config.c:1563:class_process_config()) marker 6 (0x2) lustre-client mount opts 00000020:01000000:1.0:1787761160.872877:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1787761160.872881:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.872882:0:5446:0:(obd_config.c:1563:class_process_config()) marker 7 (0x1) lustre-MDT0000 add mdc 00000020:00000080:1.0:1787761160.872885:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:1.0:1787761160.872890:0:5446:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.123@tcp to nid 0x20000c0a8cb7b (192.168.203.123@tcp) 00000020:01000000:1.0:1787761160.872918:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0000-mdc-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.872920:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1787761160.872931:0:5446:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0000-mdc-ffff9ac4523c1000 (000000006a5cb8b3) 00000020:00000080:1.0:1787761160.872948:0:5446:0:(obd_config.c:720:class_attach_name()) OBD: dev 3 attached type mdc with refcount 2 00000020:01000000:1.0:1787761160.872953:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0000-mdc-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.872955:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:1.0:1787761160.873050:0:5446:0:(ldlm_lib.c:102:import_set_conn()) imp 0000000026d00688@lustre-MDT0000-mdc-ffff9ac4523c1000: add connection 192.168.203.123@tcp at head 00010000:00010000:1.0:1787761160.875389:0:5446:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0000-mdc-ffff9ac4523c1000-1 is initialized 00000040:01000000:1.0:1787761160.875659:0:5446:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0000-mdc-ffff9ac4523c1000 ctxt 13 is initialized 00000020:00000080:1.0:1787761160.875850:0:5446:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0000-mdc-ffff9ac4523c1000 (uuid 522fbd36-e7cb-44ce-913c-7337604532e0) 00000020:01000000:1.0:1787761160.875865:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.875869:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:1.0:1787761160.875891:0:5446:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0000_UUID. index 0 00000020:01000000:1.0:1787761160.875929:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.875933:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.875934:0:5446:0:(obd_config.c:1563:class_process_config()) marker 7 (0x2) lustre-MDT0000 add mdc 00000020:01000000:1.0:1787761160.875937:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:1.0:1787761160.875940:0:5446:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #9 00000020:01000000:1.0:1787761160.875949:0:5446:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1787761160.875951:0:5446:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1787761160.875952:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:1.0:1787761160.875955:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.875956:0:5446:0:(obd_config.c:1563:class_process_config()) marker 9 (0x6) lustre sys.timeout 00000020:01000000:1.0:1787761160.875958:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:1.0:1787761160.875963:0:5446:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.203.123@tcp:/lustre 00000020:00000080:1.0:1787761160.875966:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.875967:0:5446:0:(obd_config.c:1563:class_process_config()) marker 13 (0x1) lustre-OST0000 add osc 00000020:00000080:1.0:1787761160.875970:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:1.0:1787761160.875974:0:5446:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.123@tcp to nid 0x20000c0a8cb7b (192.168.203.123@tcp) 00000020:01000000:1.0:1787761160.875998:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.876001:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1787761160.876011:0:5446:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0000-osc-ffff9ac4523c1000 (000000000e9e0aff) 00000020:00000080:1.0:1787761160.876026:0:5446:0:(obd_config.c:720:class_attach_name()) OBD: dev 4 attached type osc with refcount 2 00000020:01000000:1.0:1787761160.876031:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.876034:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:1.0:1787761160.876102:0:5446:0:(ldlm_lib.c:102:import_set_conn()) imp 00000000eca1b012@lustre-OST0000-osc-ffff9ac4523c1000: add connection 192.168.203.123@tcp at head 00010000:00010000:1.0:1787761160.876316:0:5446:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffff9ac4523c1000-2 is initialized 00000020:00000080:1.0:1787761160.884128:0:5446:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0000-osc-ffff9ac4523c1000 (uuid 522fbd36-e7cb-44ce-913c-7337604532e0) 00000020:01000000:1.0:1787761160.884152:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.884157:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:1.0:1787761160.884184:0:5446:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:1.0:1787761160.884204:0:5446:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff9ac4523c1000: Adding target lustre-OST0000_UUID 00020000:01000000:1.0:1787761160.884214:0:5446:0:(lov_obd.c:532:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:1.0:1787761160.884231:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.884235:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.884237:0:5446:0:(obd_config.c:1563:class_process_config()) marker 13 (0x2) lustre-OST0000 add osc 00000020:01000000:1.0:1787761160.884241:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:1.0:1787761160.884244:0:5446:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #16 00000020:01000000:1.0:1787761160.884246:0:5446:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1787761160.884248:0:5446:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1787761160.884256:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:1.0:1787761160.884259:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.884261:0:5446:0:(obd_config.c:1563:class_process_config()) marker 16 (0x6) lustre sys.timeout 00000020:01000000:1.0:1787761160.884263:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:1.0:1787761160.884268:0:5446:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.203.123@tcp:/lustre 00000020:00000080:1.0:1787761160.884271:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.884273:0:5446:0:(obd_config.c:1563:class_process_config()) marker 19 (0x1) lustre-OST0001 add osc 00000020:00000080:1.0:1787761160.884368:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:1.0:1787761160.884375:0:5446:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.123@tcp to nid 0x20000c0a8cb7b (192.168.203.123@tcp) 00000020:01000000:1.0:1787761160.884407:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.884410:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1787761160.884430:0:5446:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0001-osc-ffff9ac4523c1000 (000000004126a850) 00000020:00000080:1.0:1787761160.884451:0:5446:0:(obd_config.c:720:class_attach_name()) OBD: dev 5 attached type osc with refcount 2 00000020:01000000:1.0:1787761160.884458:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.884461:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:1.0:1787761160.884567:0:5446:0:(ldlm_lib.c:102:import_set_conn()) imp 0000000070c4862a@lustre-OST0001-osc-ffff9ac4523c1000: add connection 192.168.203.123@tcp at head 00010000:00010000:1.0:1787761160.884862:0:5446:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffff9ac4523c1000-3 is initialized 00000020:00000080:1.0:1787761160.885426:0:5446:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0001-osc-ffff9ac4523c1000 (uuid 522fbd36-e7cb-44ce-913c-7337604532e0) 00000020:01000000:1.0:1787761160.885436:0:5446:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.885440:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:1.0:1787761160.885457:0:5446:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:1.0:1787761160.885464:0:5446:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff9ac4523c1000: Adding target lustre-OST0001_UUID 00020000:01000000:1.0:1787761160.885467:0:5446:0:(lov_obd.c:532:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:1.0:1787761160.885476:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.885479:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.885481:0:5446:0:(obd_config.c:1563:class_process_config()) marker 19 (0x2) lustre-OST0001 add osc 00000020:01000000:1.0:1787761160.885484:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1787761160.885487:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.885489:0:5446:0:(obd_config.c:1563:class_process_config()) marker 22 (0x1) lustre sys.timeout 00000020:00000080:1.0:1787761160.885492:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf009 00000020:00000080:1.0:1787761160.885493:0:5446:0:(obd_config.c:1537:class_process_config()) changing lustre timeout from 100 to 20 00000100:00100000:1.0:1787761160.885564:0:5446:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000009c67219c pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5446:1874603274142720:192.168.203.123@tcp:502: 00000100:00100000:1.0:1787761160.885787:0:5446:0:(client.c:2715:ptlrpc_set_wait()) set 00000000ed797c24 going to sleep for 11 seconds 00000100:00100000:0.0:1787761160.893194:0:5446:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000009c67219c pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5446:1874603274142720:192.168.203.123@tcp:502: 00000020:01000000:0.0:1787761160.893254:0:5446:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1787761160.893264:0:5446:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1787761160.893267:0:5446:0:(obd_config.c:1563:class_process_config()) marker 22 (0x2) lustre sys.timeout 00000040:00080000:0.0:1787761160.893272:0:5446:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 39 count 39 00000020:01000000:2.0:1787761160.894487:0:5427:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 1-38 (rc=0) 10000000:01000000:2.0:1787761160.894512:0:5427:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1787761160.894552:0:5427:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761160.894565:0:5427:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: 0000000056ebb5e1/0x88466b5949dc2e47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787761160.894573:0:5427:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9ac4523c1000 from 1 10000000:01000000:2.0:1787761160.894576:0:5427:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1787761160.894615:0:5427:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761160.894622:0:5427:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761160.894631:0:5427:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787761160.894663:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000007c7e6d5c pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142848:192.168.203.123@tcp:101: 00000100:00100000:2.0:1787761160.894766:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000337503cf going to sleep for 11 seconds 00000100:00100000:3.0:1787761160.905636:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000007c7e6d5c pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142848:192.168.203.123@tcp:101: 00010000:00010000:3.0:1787761160.905668:0:5427:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245d6 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.905680:0:5427:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e245d6 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761160.906469:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142976:192.168.203.123@tcp:256: 00000100:00100000:3.0:1787761160.906660:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000ab6c91ab going to sleep for 11 seconds 00000100:00100000:3.0:1787761160.914776:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274142976:192.168.203.123@tcp:256: 10000000:01000000:3.0:1787761160.915598:0:5427:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1787761160.915627:0:5427:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245d6 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.915640:0:5427:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: 000000001622a803/0x88466b5949dc2e71 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245d6 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787761160.915649:0:5427:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff9ac4523c1000 from 1 10000000:01000000:3.0:1787761160.915663:0:5427:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1787761160.915701:0:5427:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: 00000000e39a7b36/0x88466b5949dc2e78 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.915709:0:5427:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: 00000000e39a7b36/0x88466b5949dc2e78 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.915719:0:5427:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: 00000000e39a7b36/0x88466b5949dc2e78 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761160.915740:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274143104:192.168.203.123@tcp:101: 00000100:00100000:3.0:1787761160.915830:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000ab6c91ab going to sleep for 11 seconds 00000100:00100000:3.0:1787761160.919299:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274143104:192.168.203.123@tcp:101: 00010000:00010000:3.0:1787761160.919332:0:5427:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: 00000000e39a7b36/0x88466b5949dc2e78 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761160.919343:0:5427:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: 00000000e39a7b36/0x88466b5949dc2e78 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761160.919424:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274143232:192.168.203.123@tcp:501: 00000100:00100000:3.0:1787761160.919495:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000ab6c91ab going to sleep for 11 seconds 00000100:00100000:3.0:1787761160.923088:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274143232:192.168.203.123@tcp:501: 00000100:00100000:3.0:1787761160.923159:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274143360:192.168.203.123@tcp:503: 00000100:00100000:3.0:1787761160.923288:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set 00000000ab6c91ab going to sleep for 11 seconds 00000100:00100000:3.0:1787761160.926315:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000f05a2f48 pname:cluuid:pid:xid:nid:opc:job mount.lustre:94b87323-e84d-4c2a-846c-789bad072fad:5427:1874603274143360:192.168.203.123@tcp:503: 00000100:00100000:1.0:1787761160.926893:0:5450:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000009c67219c pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5450:1874603274143488:192.168.203.123@tcp:502: 00000100:00100000:1.0:1787761160.927000:0:5450:0:(client.c:2715:ptlrpc_set_wait()) set 00000000571913d7 going to sleep for 11 seconds 00000100:00100000:1.0:1787761160.932455:0:5450:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000009c67219c pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5450:1874603274143488:192.168.203.123@tcp:502: 00000020:01000000:1.0:1787761160.932507:0:5450:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1787761160.932517:0:5450:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.932520:0:5450:0:(obd_config.c:1563:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:1.0:1787761160.932526:0:5450:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff9ac4523c1000 00000020:00000080:1.0:1787761160.932557:0:5450:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:1.0:1787761160.932613:0:5450:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451d47840 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5450:1874603274143616:192.168.203.123@tcp:502: 00000100:00100000:1.0:1787761160.932698:0:5450:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac446ac3180 going to sleep for 11 seconds 00000100:00100000:1.0:1787761160.935198:0:5450:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451d47840 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5450:1874603274143616:192.168.203.123@tcp:502: 00000020:01000000:1.0:1787761160.935257:0:5450:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1787761160.935268:0:5450:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1787761160.935271:0:5450:0:(obd_config.c:1563:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000040:00080000:1.0:1787761160.935275:0:5450:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 4 count 4 00000020:01000000:2.0:1787761160.936260:0:5427:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 1-3 (rc=0) 10000000:01000000:2.0:1787761160.936302:0:5427:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:2.0:1787761160.936362:0:5427:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761160.936376:0:5427:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:01000000:2.0:1787761160.936397:0:5427:0:(llite_lib.c:1617:ll_fill_super()) Found profile lustre-client: mdc=lustre-clilmv osc=lustre-clilov 00000020:00000080:2.0:1787761160.936422:0:5427:0:(genops.c:1385:class_connect()) connect: client 522fbd36-e7cb-44ce-913c-7337604532e0, cookie 0x88466b5949dc2e7f 00800000:01000000:2.0:1787761160.936443:0:5427:0:(lmv_obd.c:515:lmv_check_connect()) Time to connect 522fbd36-e7cb-44ce-913c-7337604532e0 to lustre-clilmv-ffff9ac4523c1000 00800000:01000000:2.0:1787761160.936450:0:5427:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0000-mdc-ffff9ac4523c1000(522fbd36-e7cb-44ce-913c-7337604532e0) - lustre-MDT0000_UUID, 522fbd36-e7cb-44ce-913c-7337604532e0 00000020:00000080:2.0:1787761160.936457:0:5427:0:(genops.c:1385:class_connect()) connect: client 522fbd36-e7cb-44ce-913c-7337604532e0, cookie 0x88466b5949dc2e86 00000100:00080000:2.0:1787761160.936467:0:5427:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 0 00000100:00080000:2.0:1787761160.936477:0:5427:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761160.936491:0:5427:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1787761160.936515:0:5427:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274143744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761160.936561:0:5427:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID 00800000:01000000:2.0:1787761160.936652:0:5427:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0000-mdc-ffff9ac4523c1000(522fbd36-e7cb-44ce-913c-7337604532e0) successfully (5) 00800000:00000004:2.0:1787761160.936656:0:5427:0:(obd_class.h:1057:obd_statfs_async()) lustre-MDT0000-mdc-ffff9ac4523c1000: age -790, max_age 0 00000080:00000004:2.0:1787761160.936681:0:5427:0:(obd_class.h:1091:obd_statfs()) lustre-clilmv-ffff9ac4523c1000: age -790, max_age 209 00800000:00000004:2.0:1787761160.936686:0:5427:0:(obd_class.h:1091:obd_statfs()) lustre-MDT0000-mdc-ffff9ac4523c1000: age -790, max_age 209 00000100:00080000:2.0:1787761160.936700:0:5427:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9ac4523bcb40 x1874603274144000/t0(0) o41->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761160.936710:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ebbc00 going to sleep for 0 seconds 00000100:00080000:2.0:1787761160.936798:0:2348:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9ac4523bc3c0 x1874603274143872/t0(0) o41->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761160.937114:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274143744:192.168.203.123@tcp:38: 00010000:00080000:3.0:1787761160.947697:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523be940 x1874603274143744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/416 e 0 to 0 dl 1787761170 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761160.947711:0:2344:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9ac4523c1000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1787761160.947720:0:2344:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to target with instance 1 00000001:01000000:2.0:1787761161.197662:0:2344:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9ac4523c1000: T10 checksum algorithm t10ip512 speed = 2503 MB/s 00000001:01000000:2.0:1787761161.447799:0:2344:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9ac4523c1000: T10 checksum algorithm t10ip4K speed = 2627 MB/s 00000001:01000000:2.0:1787761161.698187:0:2344:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9ac4523c1000: T10 checksum algorithm t10crc512 speed = 431 MB/s 00010000:00010000:1.0:1787761161.868657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:01000000:1.0:1787761161.951202:0:2344:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9ac4523c1000: T10 checksum algorithm t10crc4K speed = 407 MB/s 00000100:00080000:1.0:1787761161.951215:0:2344:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffff9ac4523c1000: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:1.0:1787761161.951231:0:2344:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:1.0:1787761161.951236:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:1.0:1787761161.955561:0:2344:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000bfeb493f): req@ffff9ac4523bcb40 x1874603274144000/t0(0) o41->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761161.955588:0:2344:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000c11e58f7): req@ffff9ac4523bc3c0 x1874603274143872/t0(0) o41->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761161.955604:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274143744:192.168.203.123@tcp:38: 00000100:00100000:2.0:1787761161.955724:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ebbc00 going to sleep for 0 seconds 00000100:00100000:2.0:1787761161.955853:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ebbc00 going to sleep for 11 seconds 00010000:00080000:3.0:1787761161.966478:0:2348:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bc3c0 x1874603274143872/t0(0) o41->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/368 e 0 to 0 dl 1787761177 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761161.966504:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:0:1874603274143872:192.168.203.123@tcp:41: 00010000:00080000:2.0:1787761161.974697:0:5427:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bcb40 x1874603274144000/t0(0) o41->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/368 e 0 to 0 dl 1787761177 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761161.974719:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:522fbd36-e7cb-44ce-913c-7337604532e0:5427:1874603274144000:192.168.203.123@tcp:41: 00800000:00000004:2.0:1787761161.974745:0:5427:0:(obd_class.h:1117:obd_statfs()) lustre-MDT0000-mdc-ffff9ac4523c1000: update ffff9ac4452a38e0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00000004:2.0:1787761161.974752:0:5427:0:(obd_class.h:1117:obd_statfs()) lustre-clilmv-ffff9ac4523c1000: update ffff9ac4452a0450 cache blocks 317407/353529 objects 1023725/1024000 00000100:00100000:2.0:1787761161.974802:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:522fbd36-e7cb-44ce-913c-7337604532e0:5427:1874603274144128:192.168.203.123@tcp:40: 00000100:00100000:2.0:1787761161.975471:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ebb180 going to sleep for 11 seconds 00010000:00080000:2.0:1787761161.997439:0:5427:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bcb40 x1874603274144128/t0(0) o40->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 440/448 e 0 to 0 dl 1787761177 ref 2 fl Rpc:RQU/209/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761161.997459:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:522fbd36-e7cb-44ce-913c-7337604532e0:5427:1874603274144128:192.168.203.123@tcp:40: 00000080:00000004:2.0:1787761161.997482:0:5427:0:(llite_lib.c:615:client_common_fill_super()) rootfid [0x200000007:0x1:0x0] 00000080:00100000:2.0:1787761161.997496:0:5427:0:(llite_lib.c:778:client_common_fill_super()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683255 ocd_grant: 2146304 00020000:01000000:2.0:1787761161.997500:0:5427:0:(lov_obd.c:186:lov_connect()) connect #0 00000020:00000080:2.0:1787761161.997511:0:5427:0:(genops.c:1385:class_connect()) connect: client 522fbd36-e7cb-44ce-913c-7337604532e0, cookie 0x88466b5949dc2e8d 00000020:00000080:2.0:1787761161.997632:0:5427:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x88466b5949dc2e94 00000100:00080000:2.0:1787761161.997641:0:5427:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 0 00000100:00080000:2.0:1787761161.997649:0:5427:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9ac4523c1000: import ffff9ac4454bb000 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761161.997662:0:5427:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683255 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1787761161.998255:0:5427:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274144256/t0(0) o8->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761161.998322:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274144256:192.168.203.123@tcp:8: 00000100:00080000:2.0:1787761161.999676:0:5427:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-OST0000_UUID 00020000:01000000:2.0:1787761162.001052:0:5427:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 0 lustre-OST0000_UUID (lustre-OST0000-osc-ffff9ac4523c1000) active 00000020:00000080:2.0:1787761162.001097:0:5427:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x88466b5949dc2e9b 00000100:00080000:2.0:1787761162.001111:0:5427:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 0 00000100:00080000:2.0:1787761162.001121:0:5427:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9ac4523c1000: import ffff9ac442b82000 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761162.001134:0:5427:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683255 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1787761162.001158:0:5427:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274144384/t0(0) o8->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761162.002982:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274144384:192.168.203.123@tcp:8: 00000100:00100000:3.0:1787761162.003831:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274144512:192.168.203.123@tcp:400: 00000100:00080000:2.0:1787761162.005921:0:5427:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-OST0001_UUID 00020000:01000000:2.0:1787761162.005944:0:5427:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 1 lustre-OST0001_UUID (lustre-OST0001-osc-ffff9ac4523c1000) active 00000100:00100000:2.0:1787761162.006027:0:5427:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job mount.lustre:522fbd36-e7cb-44ce-913c-7337604532e0:5427:1874603274144640:192.168.203.123@tcp:33: 00000100:00100000:2.0:1787761162.006170:0:5427:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ebb180 going to sleep for 11 seconds 00000100:00100000:3.0:1787761162.028003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274144512:192.168.203.123@tcp:400: 00010000:00080000:0.0:1787761162.028137:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bd680 x1874603274144384/t0(0) o8->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 520/416 e 0 to 0 dl 1787761172 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761162.028150:0:2344:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9ac4523c1000: committing for last_committed 0 gen 1 00000100:00080000:0.0:1787761162.028161:0:2344:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff9ac4523c1000: connect to target with instance 1 00000100:00080000:0.0:1787761162.028173:0:2344:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff9ac4523c1000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:0.0:1787761162.028189:0:2344:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:0.0:1787761162.028195:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac442b82000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:0.0:1787761162.028204:0:2344:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:0.0:1787761162.028212:0:2344:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:0.0:1787761162.028218:0:2344:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:0.0:1787761162.028242:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274144384:192.168.203.123@tcp:8: 00010000:00080000:0.0:1787761162.028264:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bcb40 x1874603274144256/t0(0) o8->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 520/416 e 0 to 0 dl 1787761171 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761162.028272:0:2344:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9ac4523c1000: committing for last_committed 0 gen 1 00000100:00080000:0.0:1787761162.028276:0:2344:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff9ac4523c1000: connect to target with instance 1 00000100:00080000:0.0:1787761162.028281:0:2344:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff9ac4523c1000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:0.0:1787761162.028286:0:2344:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:0.0:1787761162.028288:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bb000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:0.0:1787761162.028309:0:2344:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:0.0:1787761162.028311:0:2344:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:0.0:1787761162.028314:0:2344:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:0.0:1787761162.028319:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274144256:192.168.203.123@tcp:8: 00010000:00080000:2.0:1787761162.080865:0:5427:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523be1c0 x1874603274144640/t0(0) o33->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:23/10 lens 448/464 e 0 to 0 dl 1787761178 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000100:00100000:2.0:1787761162.080892:0:5427:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job mount.lustre:522fbd36-e7cb-44ce-913c-7337604532e0:5427:1874603274144640:192.168.203.123@tcp:33: 00000080:00200000:2.0:1787761162.081621:0:5427:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00200000:2.0:1787761162.081632:0:5427:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:02000400:2.0:1787761162.081720:0:5427:0:(llite_lib.c:1656:ll_fill_super()) Mounted lustre-client - version 2.17.57_119_g5069d9c 00000080:00000004:2.0:1787761162.089485:0:5427:0:(super25.c:182:lustre_fill_super()) 192.168.203.123@tcp:/lustre: Mount complete 00000080:00200008:3.0:1787761162.435756:0:5467:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787761162.435784:0:5467:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761162.435816:0:5467:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761162.435896:0:5467:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 5467 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.435909:0:5467:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5467 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.435920:0:5467:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5467 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761162.435950:0:5467:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job mkdir:522fbd36-e7cb-44ce-913c-7337604532e0:5467:1874603274144768:192.168.203.123@tcp:101: 00000100:00100000:3.0:1787761162.436084:0:5467:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc900 going to sleep for 11 seconds 00000100:00100000:3.0:1787761162.439604:0:5467:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job mkdir:522fbd36-e7cb-44ce-913c-7337604532e0:5467:1874603274144768:192.168.203.123@tcp:101: 00010000:00010000:3.0:1787761162.439636:0:5467:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439678:0:5467:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439692:0:5467:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439708:0:5467:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439724:0:5467:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439733:0:5467:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439742:0:5467:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787761162.439754:0:5467:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9ac451c52580 x1874603274144768/t0(0) o101->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 576/688 e 0 to 0 dl 1787761178 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787761162.439780:0:5467:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439795:0:5467:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9ac448a01000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787761162.439802:0:5467:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787761162.439835:0:5467:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787761162.439846:0:5467:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:3.0:1787761162.439853:0:5467:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761162.439856:0:5467:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d03c53cc0 00010000:00010000:3.0:1787761162.439860:0:5467:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761162.439868:0:5467:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761162.884914:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761163.908884:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1787761164.877802:0:5520:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787761164.877817:0:5520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:0.0:1787761164.877842:0:5520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787761164.877862:0:5520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787761164.877879:0:5520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787761164.877888:0:5520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787761164.877896:0:5520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:0.0:1787761164.877900:0:5520:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:0.0:1787761164.877907:0:5520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01863cc0 00010000:00010000:0.0:1787761164.877911:0:5520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787761164.877920:0:5520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:0.0:1787761164.878836:0:5520:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787761164.878844:0:5520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:0.0:1787761164.878854:0:5520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787761164.878864:0:5520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:0.0:1787761164.878882:0:5520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:0.0:1787761164.878890:0:5520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787761164.878896:0:5520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:0.0:1787761164.878899:0:5520:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:0.0:1787761164.878904:0:5520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01863cc0 00010000:00010000:0.0:1787761164.878907:0:5520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:0.0:1787761164.878916:0:5520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:0.0:1787761164.879019:0:5520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:0.0:1787761164.879026:0:5520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787761164.879034:0:5520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1787761164.879042:0:5520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:0.0:1787761164.879049:0:5520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787761164.879053:0:5520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:0.0:1787761164.879055:0:5520:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:0.0:1787761164.879059:0:5520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01863bb0 00010000:00010000:0.0:1787761164.879062:0:5520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1787761164.879076:0:5520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:0.0:1787761164.879085:0:5520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08), inode mode 41ed mask 44 00000080:00200008:0.0:1787761164.879093:0:5520:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), flags 100000 00000080:00200080:0.0:1787761164.879293:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:0.0:1787761164.879308:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac446960400 uarg=0000000000000000 00800000:00000004:0.0:1787761164.879315:0:5520:0:(obd_class.h:1091:obd_statfs()) lustre-MDT0000-mdc-ffff9ac4523c1000: age 211, max_age 213 00000100:00100000:0.0:1787761164.879356:0:5520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87840 pname:cluuid:pid:xid:nid:opc:job lfs:522fbd36-e7cb-44ce-913c-7337604532e0:5520:1874603274144896:192.168.203.123@tcp:41: 00000100:00100000:0.0:1787761164.879626:0:5520:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac451e4ab40 going to sleep for 11 seconds 00000100:00100000:0.0:1787761164.885182:0:5520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87840 pname:cluuid:pid:xid:nid:opc:job lfs:522fbd36-e7cb-44ce-913c-7337604532e0:5520:1874603274144896:192.168.203.123@tcp:41: 00800000:00000004:0.0:1787761164.885223:0:5520:0:(obd_class.h:1117:obd_statfs()) lustre-MDT0000-mdc-ffff9ac4523c1000: update ffff9ac4452a38e0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00200080:3.0:1787761164.885649:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885667:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885675:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885679:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885684:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885688:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885693:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885697:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885702:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885705:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885709:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885712:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885716:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885719:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885723:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885727:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885742:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885747:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885880:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885888:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885893:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885897:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885901:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885905:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885909:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885912:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885916:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885919:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885923:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885926:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885930:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885934:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885938:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885941:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885945:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885949:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885953:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885956:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885960:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885964:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885968:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885978:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.885983:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.885987:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886872:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886883:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886888:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886893:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886897:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886900:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886904:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886907:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886911:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886915:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886919:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886922:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886926:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886930:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886934:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886937:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886941:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886945:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886948:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886952:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886956:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886959:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886975:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.886980:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.886984:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887065:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887075:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887081:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887086:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887089:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887094:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887098:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887102:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887106:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887110:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887114:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887118:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887122:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887126:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887129:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887133:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887137:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887141:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887145:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887149:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887152:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887156:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887167:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887173:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887177:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887180:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887183:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887187:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887190:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887194:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887197:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887201:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887205:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887209:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887212:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887216:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887218:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887222:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887225:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887228:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887232:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887235:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887239:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887242:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887246:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887250:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887253:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887266:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887271:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887275:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887278:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887282:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887286:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887290:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887294:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887298:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887301:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887306:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00800000:00000080:3.0:1787761164.887310:0:5520:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00000080:00200080:3.0:1787761164.887314:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.887319:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac444fd1400 uarg=0000000000000000 00020000:00000004:3.0:1787761164.887327:0:5520:0:(obd_class.h:1091:obd_statfs()) lustre-OST0000-osc-ffff9ac4523c1000: age -790, max_age 213 00000100:00100000:3.0:1787761164.887372:0:5520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job lfs:522fbd36-e7cb-44ce-913c-7337604532e0:5520:1874603274145024:192.168.203.123@tcp:13: 00000100:00100000:3.0:1787761164.887481:0:5520:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc540 going to sleep for 11 seconds 00010000:00080000:3.0:1787761164.892203:0:5520:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451c50780 x1874603274145024/t0(0) o13->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:7/4 lens 224/368 e 0 to 0 dl 1787761180 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761164.892225:0:5520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job lfs:522fbd36-e7cb-44ce-913c-7337604532e0:5520:1874603274145024:192.168.203.123@tcp:13: 00020000:00000004:3.0:1787761164.892250:0:5520:0:(obd_class.h:1117:obd_statfs()) lustre-OST0000-osc-ffff9ac4523c1000: update ffff9ac4452a1e98 cache blocks 901403/958279 objects 261840/262144 00000080:00200080:3.0:1787761164.892326:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.892344:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00020000:00000004:3.0:1787761164.892351:0:5520:0:(obd_class.h:1091:obd_statfs()) lustre-OST0001-osc-ffff9ac4523c1000: age -790, max_age 213 00000100:00100000:3.0:1787761164.892397:0:5520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job lfs:522fbd36-e7cb-44ce-913c-7337604532e0:5520:1874603274145152:192.168.203.123@tcp:13: 00000100:00100000:3.0:1787761164.892472:0:5520:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc540 going to sleep for 11 seconds 00010000:00080000:3.0:1787761164.897739:0:5520:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451c50780 x1874603274145152/t0(0) o13->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:7/4 lens 224/368 e 0 to 0 dl 1787761180 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761164.897773:0:5520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job lfs:522fbd36-e7cb-44ce-913c-7337604532e0:5520:1874603274145152:192.168.203.123@tcp:13: 00020000:00000004:3.0:1787761164.897798:0:5520:0:(obd_class.h:1117:obd_statfs()) lustre-OST0001-osc-ffff9ac4523c1000: update ffff9ac4495538e0 cache blocks 901403/958279 objects 261840/262144 00000080:00200080:3.0:1787761164.897872:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897889:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897897:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897902:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897906:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897909:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897913:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897916:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897920:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897924:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897927:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897930:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897934:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897937:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897941:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897945:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897949:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897951:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897974:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897979:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.897983:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.897986:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899089:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899099:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899104:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899108:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899112:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899115:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899118:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899121:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899125:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899127:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899131:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899134:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899138:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899141:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899145:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899148:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899151:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899155:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899158:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899162:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899165:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899178:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899183:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899187:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899191:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899195:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899198:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899202:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899206:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899209:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899213:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899216:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899219:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899222:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899226:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899229:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899232:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899235:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899238:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899242:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899245:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899248:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899252:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899255:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899259:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899262:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899273:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899277:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899282:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899285:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899288:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899291:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899295:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899298:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899302:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899305:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899309:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899312:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899316:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899320:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899324:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899327:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899331:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899335:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899339:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899342:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899347:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899350:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899354:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899357:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899361:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899371:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899376:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899380:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899383:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899387:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899390:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899394:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899397:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899401:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899404:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899407:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899411:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899414:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899418:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899422:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899426:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899429:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899433:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899437:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899440:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899443:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899447:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899450:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899454:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899458:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899467:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899472:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899475:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899479:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899482:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899485:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200080:3.0:1787761164.899489:0:5520:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08) cmd=c00866a4 arg=7ffea6080f10 00020000:00000080:3.0:1787761164.899492:0:5520:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9ac4523c1000: cmd=c00866a4 len=592 karg=ffff9ac447821c00 uarg=0000000000000000 00000080:00200008:3.0:1787761164.899505:0:5520:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), flags 100000 00010000:00010000:3.0:1787761164.934296:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761165.963265:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761166.984723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761167.174364:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274145280:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761167.174504:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274145536:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761167.174565:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1787761167.174572:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274145408:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761167.174597:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274145664:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761167.184921:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274145408:192.168.203.123@tcp:400: 00010000:00080000:3.0:1787761167.184941:0:2348:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bcf00 x1874603274145664/t0(0) o400->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761183 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761167.184953:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274145664:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761167.184963:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274145280:192.168.203.123@tcp:400: 00010000:00080000:3.0:1787761167.184973:0:2348:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac4523bc000 x1874603274145536/t0(0) o400->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761183 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761167.184996:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274145536:192.168.203.123@tcp:400: 00000001:02000400:0.0:1787761167.553704:0:5709:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:2.0:1787761168.007326:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761169.029347:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761170.052882:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761171.078655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761171.434078:0:2337:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.123@tcp (source: 12345-192.168.203.123@tcp) 00000100:00100000:0.0:1787761171.434121:0:2337:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.123@tcp (source: 12345-192.168.203.123@tcp) 00000100:00100000:0.0:1787761171.434184:0:5429:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874603295140864 00000100:00100000:0.0:1787761171.434193:0:5429:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874603295140864 00000100:00100000:0.0:1787761171.434234:0:5429:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874603295140992 00000100:00100000:0.0:1787761171.434236:0:5429:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874603295140992 00000100:00100000:0.0:1787761171.434249:0:5429:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.123@tcp, seq: 0 00000100:00100000:0.0:1787761171.434256:0:5429:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9ac451f87840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:94b87323-e84d-4c2a-846c-789bad072fad+4:8827:x1874603295140864:12345-192.168.203.123@tcp:104: 00010000:00010000:0.0:1787761171.434274:0:5429:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787761171.434352:0:5429:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1787761171.434358:0:5429:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9ac451f87840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:94b87323-e84d-4c2a-846c-789bad072fad+4:8827:x1874603295140864:12345-192.168.203.123@tcp:104: Request processed in 102us (291us total) trans 0 rc 0/0 00000100:00100000:0.0:1787761171.434365:0:5429:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.123@tcp, seq: 0 00000100:00100000:0.0:1787761171.434383:0:5429:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.123@tcp, seq: 1 00000100:00100000:0.0:1787761171.434389:0:5429:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9ac451f86580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:94b87323-e84d-4c2a-846c-789bad072fad+3:8827:x1874603295140992:12345-192.168.203.123@tcp:104: 00010000:00010000:0.0:1787761171.434397:0:5429:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787761171.434429:0:5429:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1787761171.434434:0:5429:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9ac451f86580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:94b87323-e84d-4c2a-846c-789bad072fad+3:8827:x1874603295140992:12345-192.168.203.123@tcp:104: Request processed in 46us (314us total) trans 0 rc 0/0 00000100:00100000:0.0:1787761171.434441:0:5429:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.123@tcp, seq: 1 00010000:00010000:3.0:1787761171.434678:0:5438:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1787761171.434686:0:5438:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434693:0:5438:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9ac4414a8c00 already unused, calling callback (ffffffffc12c2c20) 10000000:00010000:3.0:1787761171.434697:0:5438:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434714:0:5438:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1787761171.434721:0:5438:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787761171.434727:0:5438:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1787761171.434768:0:5438:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434775:0:5438:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787761171.434796:0:5438:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434803:0:5438:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2e47 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245cf expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434818:0:5438:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787761171.434828:0:5438:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434833:0:5438:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9ac440030400 already unused, calling callback (ffffffffc12c2c20) 10000000:00010000:3.0:1787761171.434835:0:5438:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434842:0:5438:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1787761171.434847:0:5438:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787761171.434852:0:5438:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1787761171.434866:0:5438:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434872:0:5438:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787761171.434877:0:5438:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434884:0:5438:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.123@tcp lock: ffff9ac440030400/0x88466b5949dc2e40 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245c8 expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761171.434896:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761171.434898:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761171.434899:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761171.434915:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274145792:192.168.203.123@tcp:103: 00000100:00100000:3.0:1787761171.434977:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787761171.434984:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274145920:192.168.203.123@tcp:103: 00010000:00010000:2.0:1787761171.436182:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761171.438477:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274145920:192.168.203.123@tcp:103: 00000100:00100000:3.0:1787761171.438492:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274145792:192.168.203.123@tcp:103: 00010000:00010000:3.0:1787761172.101026:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761172.293675:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274146176:192.168.203.123@tcp:400: 00000100:00100000:0.0:1787761172.293781:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274146432:192.168.203.123@tcp:400: 00000100:00100000:0.0:1787761172.293822:0:2345:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1787761172.293828:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274146048:192.168.203.123@tcp:400: 00000100:00100000:0.0:1787761172.293853:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274146304:192.168.203.123@tcp:400: 10000000:01000000:3.0:1787761172.297063:0:5442:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1787761172.297071:0:5442:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff9ac4523c1000 from 39 10000000:01000000:3.0:1787761172.297077:0:5442:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1787761172.297127:0:5442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.297140:0:5442:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.297151:0:5442:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761172.297174:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274146560:192.168.203.123@tcp:101: 00000100:00100000:3.0:1787761172.297280:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc840 going to sleep for 11 seconds 00000100:00100000:0.0:1787761172.303869:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274146048:192.168.203.123@tcp:400: 00010000:00080000:0.0:1787761172.303891:0:2345:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451f85680 x1874603274146304/t0(0) o400->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761188 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761172.303904:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274146304:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761172.303906:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274146560:192.168.203.123@tcp:101: 00000100:00100000:0.0:1787761172.303932:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274146176:192.168.203.123@tcp:400: 00010000:00010000:3.0:1787761172.303932:0:5442:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e2460e expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00080000:0.0:1787761172.303941:0:2345:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451f86d00 x1874603274146432/t0(0) o400->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761188 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1787761172.303944:0:5442:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e2460e expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787761172.303950:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274146432:192.168.203.123@tcp:400: 00000100:00100000:3.0:1787761172.304034:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274146688:192.168.203.123@tcp:501: 00000100:00100000:3.0:1787761172.304126:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc840 going to sleep for 11 seconds 00000100:00100000:3.0:1787761172.309466:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274146688:192.168.203.123@tcp:501: 00000100:00100000:3.0:1787761172.309566:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274146816:192.168.203.123@tcp:503: 00000100:00100000:3.0:1787761172.309833:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc840 going to sleep for 11 seconds 00000100:00100000:3.0:1787761172.317258:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274146816:192.168.203.123@tcp:503: 00000100:00100000:1.0:1787761172.325559:0:5908:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451d47840 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5908:1874603274146944:192.168.203.123@tcp:502: 00000100:00100000:1.0:1787761172.325681:0:5908:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4607bd600 going to sleep for 11 seconds 00000100:00100000:2.0:1787761172.332266:0:5908:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451d47840 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5908:1874603274146944:192.168.203.123@tcp:502: 00000020:01000000:2.0:1787761172.332326:0:5908:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1787761172.332337:0:5908:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787761172.332339:0:5908:0:(obd_config.c:1563:class_process_config()) marker 26 (0x1) lustre sys.jobid_var 00000020:00000080:2.0:1787761172.332346:0:5908:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00f 00000100:00100000:2.0:1787761172.332388:0:5908:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5908:1874603274147072:192.168.203.123@tcp:502:llog_process_th.0 00000100:00100000:2.0:1787761172.332484:0:5908:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460e70d80 going to sleep for 11 seconds 00000100:00100000:2.0:1787761172.336732:0:5908:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5908:1874603274147072:192.168.203.123@tcp:502:llog_process_th.0 00000020:01000000:2.0:1787761172.336776:0:5908:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787761172.336784:0:5908:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787761172.336787:0:5908:0:(obd_config.c:1563:class_process_config()) marker 26 (0x2) lustre sys.jobid_var 00000040:00080000:2.0:1787761172.336802:0:5908:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 42 count 42 00000020:01000000:3.0:1787761172.337025:0:5442:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 39-41 (rc=0) 10000000:01000000:3.0:1787761172.337050:0:5442:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1787761172.337061:0:5442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e2460e expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.337073:0:5442:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8c00/0x88466b5949dc2ea9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e2460e expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787761172.337083:0:5442:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:3.0:1787761172.337085:0:5442:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff9ac4601c9a48 from 1 10000000:01000000:3.0:1787761172.337088:0:5442:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:3.0:1787761172.337119:0:5442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.337127:0:5442:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.337136:0:5442:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761172.337157:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147200:192.168.203.123@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1787761172.337234:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc840 going to sleep for 11 seconds 00000100:00100000:3.0:1787761172.340635:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147200:192.168.203.123@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1787761172.340675:0:5442:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e24615 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.340718:0:5442:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e24615 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761172.340817:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147328:192.168.203.123@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1787761172.340920:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460ecc840 going to sleep for 11 seconds 00000100:00100000:3.0:1787761172.350557:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147328:192.168.203.123@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1787761172.350617:0:5442:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1787761172.350629:0:5442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e24615 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761172.350639:0:5442:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: ffff9ac4414a9800/0x88466b5949dc2eb0 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e24615 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761173.125791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761173.865244:0:2337:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.123@tcp (source: 12345-192.168.203.123@tcp) 00000100:00100000:1.0:1787761173.868967:0:5429:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874603295147136 00000100:00100000:1.0:1787761173.868986:0:5429:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874603295147136 00000100:00100000:1.0:1787761173.869101:0:5429:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.123@tcp, seq: 2 00000100:00100000:1.0:1787761173.869116:0:5429:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9ac451f86d00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:94b87323-e84d-4c2a-846c-789bad072fad+3:8886:x1874603295147136:12345-192.168.203.123@tcp:104: 00010000:00010000:1.0:1787761173.869137:0:5429:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787761173.869256:0:5429:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1787761173.869266:0:5429:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9ac451f86d00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:94b87323-e84d-4c2a-846c-789bad072fad+3:8886:x1874603295147136:12345-192.168.203.123@tcp:104: Request processed in 155us (4031us total) trans 0 rc 0/0 00000100:00100000:1.0:1787761173.869274:0:5429:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.123@tcp, seq: 2 00010000:00010000:3.0:1787761173.869653:0:5438:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787761173.869663:0:5438:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761173.869673:0:5438:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9ac4414a8400 already unused, calling callback (ffffffffc12c2c20) 10000000:00010000:3.0:1787761173.869677:0:5438:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761173.869698:0:5438:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1787761173.869708:0:5438:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787761173.869739:0:5438:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1787761173.869825:0:5438:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761173.869834:0:5438:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787761173.869863:0:5438:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761173.869872:0:5438:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2e78 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e245dd expref: -99 pid: 5427 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761173.869894:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761173.869897:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761173.869899:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761173.869922:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274147456:192.168.203.123@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787761173.884017:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274147456:192.168.203.123@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1787761174.151755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761175.172783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1787761175.428674:0:5442:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1787761175.428690:0:5442:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff9ac4523c1000 from 4 10000000:01000000:2.0:1787761175.428696:0:5442:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1787761175.428751:0:5442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761175.428764:0:5442:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761175.428775:0:5442:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787761175.428805:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147584:192.168.203.123@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1787761175.428911:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460e706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787761175.441856:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147584:192.168.203.123@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1787761175.441905:0:5442:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761175.441919:0:5442:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787761175.441997:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147712:192.168.203.123@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1787761175.442083:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460e706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787761175.467708:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147712:192.168.203.123@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1787761175.467774:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147840:192.168.203.123@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1787761175.467874:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460e706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787761175.472407:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274147840:192.168.203.123@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1787761175.473414:0:5971:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5971:1874603274147968:192.168.203.123@tcp:502:llog_process_th.0 00000100:00100000:2.0:1787761175.473517:0:5971:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460e70300 going to sleep for 11 seconds 00000100:00100000:2.0:1787761175.485019:0:5971:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5971:1874603274147968:192.168.203.123@tcp:502:llog_process_th.0 00000020:01000000:2.0:1787761175.485073:0:5971:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1787761175.485084:0:5971:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787761175.485086:0:5971:0:(obd_config.c:1563:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:2.0:1787761175.485093:0:5971:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff9ac4523c1000 00000020:00000080:2.0:1787761175.485096:0:5971:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:2.0:1787761175.485250:0:5971:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5971:1874603274148096:192.168.203.123@tcp:502:llog_process_th.0 00000100:00100000:2.0:1787761175.485342:0:5971:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac460e70300 going to sleep for 11 seconds 00000100:00100000:2.0:1787761175.494126:0:5971:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:94b87323-e84d-4c2a-846c-789bad072fad:5971:1874603274148096:192.168.203.123@tcp:502:llog_process_th.0 00000020:01000000:2.0:1787761175.494187:0:5971:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787761175.494195:0:5971:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787761175.494198:0:5971:0:(obd_config.c:1563:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:2.0:1787761175.494203:0:5971:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:3.0:1787761175.508402:0:5442:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:3.0:1787761175.508434:0:5442:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1787761175.508445:0:5442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761175.508459:0:5442:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761176.196722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761177.222066:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761177.412952:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274148224:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761177.413886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274148480:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761177.413941:0:2347:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787761177.413949:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274148352:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761177.413986:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274148608:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761177.417025:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274148352:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761177.417042:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274148224:192.168.203.123@tcp:400:kworker.0 00010000:00080000:3.0:1787761177.417056:0:2347:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451c50b40 x1874603274148480/t0(0) o400->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761193 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761177.417067:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274148480:192.168.203.123@tcp:400:kworker.0 00010000:00080000:3.0:1787761177.421717:0:2347:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451c52940 x1874603274148608/t0(0) o400->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761193 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761177.421737:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274148608:192.168.203.123@tcp:400:kworker.0 00010000:00010000:0.0:1787761178.246209:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761179.274612:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761180.294115:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761181.317601:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787761181.586445:0:6269:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg323-client.virtnet: executing check_logdir /tmp/testlogs/ 00010000:00010000:0.0:1787761182.343876:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761182.532897:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274148864:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761182.533056:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274149120:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761182.533099:0:2345:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1787761182.533106:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451d47840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274148736:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761182.533135:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274148992:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761182.539589:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451d47840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274148736:192.168.203.123@tcp:400:kworker.0 00010000:00080000:0.0:1787761182.539620:0:2345:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac447840b40 x1874603274148992/t0(0) o400->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761198 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761182.539647:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274148992:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761182.539662:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274148864:192.168.203.123@tcp:400:kworker.0 00010000:00080000:0.0:1787761182.539673:0:2345:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac447842940 x1874603274149120/t0(0) o400->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761198 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761182.539682:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274149120:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761183.364745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761184.396465:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761185.413432:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761186.436807:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761187.461684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761187.652915:0:8:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bb000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1787761187.652922:0:8:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff9ac4523c1000: disconnect after 23s idle 00000100:00100000:2.0:1787761187.657999:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274149248:192.168.203.123@tcp:400:kworker.0 00000100:00100000:2.0:1787761187.658138:0:2347:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787761187.658147:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274149376:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761187.658731:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274149504:192.168.203.123@tcp:9:kworker.0 00000100:00080000:3.0:1787761187.658817:0:8:0:(import.c:65:import_set_state_nolock()) ffff9ac442b82000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1787761187.658821:0:8:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff9ac4523c1000: disconnect after 23s idle 00000100:00100000:1.0:1787761187.662949:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274149632:192.168.203.123@tcp:9:kworker.0 00010000:00080000:0.0:1787761187.676064:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451c530c0 x1874603274149632/t0(0) o9->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761197 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761187.676101:0:2344:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff9ac451c530c0 x1874603274149632/t0(0) o9->lustre-OST0001-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761197 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761187.676127:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac442b82000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1787761187.676132:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274149632:192.168.203.123@tcp:9:kworker.0 00010000:00080000:0.0:1787761187.676148:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9ac451c52d00 x1874603274149504/t0(0) o9->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761197 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761187.676160:0:2344:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff9ac451c52d00 x1874603274149504/t0(0) o9->lustre-OST0000-osc-ffff9ac4523c1000@192.168.203.123@tcp:28/4 lens 224/224 e 0 to 0 dl 1787761197 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761187.676166:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bb000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1787761187.676169:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274149504:192.168.203.123@tcp:9:kworker.0 00000100:00100000:3.0:1787761187.677284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274149376:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761187.677300:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274149248:192.168.203.123@tcp:400:kworker.0 00000001:02000400:2.0:1787761187.833979:0:6551:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg323-client.virtnet: executing yml_node 00010000:00010000:3.0:1787761188.486414:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761189.511155:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761190.535669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761191.558143:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761192.582606:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761192.772926:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274149760:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761192.773040:0:2346:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787761192.773048:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274149888:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761192.775674:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274149888:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761192.778446:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274149760:192.168.203.123@tcp:400:kworker.0 00000001:02000400:2.0:1787761193.022711:0:6989:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.57.119 00010000:00010000:1.0:1787761193.604780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761194.632324:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1787761195.602925:0:7143:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.57.119 00010000:00010000:2.0:1787761195.662263:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761196.677761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761197.700693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761197.892903:0:2345:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787761197.892929:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274150016:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761197.893175:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274150144:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761197.898460:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274150016:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761197.902272:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274150144:192.168.203.123@tcp:400:kworker.0 00000001:02000400:3.0:1787761198.167658:0:7298:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.57.119 00010000:00010000:3.0:1787761198.725853:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761199.767389:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1787761199.820790:0:7443:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: sanity-compr ============----- Wed Aug 26 12:19:58 EDT 2026 00010000:00010000:1.0:1787761200.772738:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761201.797057:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761202.825452:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761203.013929:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274150400:192.168.203.123@tcp:400:kworker.0 00000100:00100000:2.0:1787761203.014060:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1787761203.014068:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274150272:192.168.203.123@tcp:400:kworker.0 00000100:00100000:2.0:1787761203.031460:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274150272:192.168.203.123@tcp:400:kworker.0 00000100:00100000:2.0:1787761203.031482:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274150400:192.168.203.123@tcp:400:kworker.0 00010000:00010000:1.0:1787761203.845338:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761204.868818:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761205.893131:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761206.918116:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761207.942614:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761208.135229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274150528:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761208.135346:0:2347:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787761208.135354:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274150656:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761208.141222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274150656:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761208.141249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274150528:192.168.203.123@tcp:400:kworker.0 00010000:00010000:0.0:1787761208.971491:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761209.989911:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761211.015873:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761212.037387:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761213.062138:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761213.253037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274150784:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761213.253226:0:2347:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787761213.253233:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274150912:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761213.260904:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274150912:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761213.260921:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274150784:192.168.203.123@tcp:400:kworker.0 00010000:00010000:3.0:1787761214.092362:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761215.108696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761216.137900:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761217.157717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761218.180815:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761218.373668:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274151168:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761218.373803:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787761218.373809:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274151040:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761218.387355:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274151040:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761218.387455:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274151168:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761219.209934:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761220.232804:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761221.252644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1787761221.983918:0:7961:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 1081 00010000:00010000:3.0:1787761222.285231:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761223.300845:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761223.494156:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274151296:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761223.494442:0:2346:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787761223.494450:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274151424:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761223.506471:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274151424:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761223.506985:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274151296:192.168.203.123@tcp:400:kworker.0 00000001:02000400:2.0:1787761223.715250:0:8104:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanity-compr: start setup 12:20:22 (1787761222) === 00000080:00200008:1.0:1787761223.759758:0:7460:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787761223.759769:0:7460:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:1.0:1787761223.759808:0:7460:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787761223.759829:0:7460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1787761223.759862:0:7460:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1787761223.759870:0:7460:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787761223.759878:0:7460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:1.0:1787761223.759881:0:7460:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:1.0:1787761223.759889:0:7460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01c1fcc0 00010000:00010000:1.0:1787761223.759893:0:7460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1787761223.759902:0:7460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200008:3.0:1787761223.788645:0:7460:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787761223.788657:0:7460:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761223.788677:0:7460:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761223.788695:0:7460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1787761223.788711:0:7460:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:3.0:1787761223.788719:0:7460:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761223.788727:0:7460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761223.788730:0:7460:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:3.0:1787761223.788737:0:7460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01c1fcc0 00010000:00010000:3.0:1787761223.788741:0:7460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1787761223.788750:0:7460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200008:1.0:1787761223.812289:0:7460:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787761223.812302:0:7460:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:1.0:1787761223.812325:0:7460:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787761223.812355:0:7460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1787761223.812372:0:7460:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1787761223.812379:0:7460:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787761223.812387:0:7460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:1.0:1787761223.812390:0:7460:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:1.0:1787761223.812397:0:7460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01c1fcc0 00010000:00010000:1.0:1787761223.812409:0:7460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1787761223.812419:0:7460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200008:3.0:1787761223.848555:0:7460:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787761223.848569:0:7460:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761223.848591:0:7460:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761223.848612:0:7460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1787761223.848636:0:7460:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:3.0:1787761223.848646:0:7460:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761223.848655:0:7460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761223.848659:0:7460:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:3.0:1787761223.848667:0:7460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01c1fcc0 00010000:00010000:3.0:1787761223.848671:0:7460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1787761223.848680:0:7460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1787761224.329024:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761225.352552:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761226.373318:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761227.405987:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761228.420787:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761228.614790:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274151552:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761228.614887:0:2346:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787761228.614895:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274151680:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761228.627474:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274151680:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761228.627490:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274151552:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761229.444709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761230.469869:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761231.743912:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787761232.506525:0:8373:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg323-client.virtnet: executing check_config_client /mnt/lustre 00010000:00010000:3.0:1787761232.774974:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761233.735383:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274151808:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761233.739014:0:2346:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787761233.739042:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274151936:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761233.758062:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274151936:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761233.758082:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274151808:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761233.797710:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761234.824583:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761235.853676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761236.872370:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761237.897845:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761238.858081:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274152192:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761238.858179:0:2345:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787761238.858186:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274152064:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761238.869005:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274152064:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761238.869027:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274152192:192.168.203.123@tcp:400:kworker.0 00010000:00010000:3.0:1787761238.920432:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761239.941951:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761240.964963:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761241.992607:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761243.015715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761243.973035:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274152320:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761243.973174:0:2347:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787761243.973181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274152448:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761243.975620:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274152320:192.168.203.123@tcp:400:kworker.0 00000100:00100000:3.0:1787761243.976174:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:522fbd36-e7cb-44ce-913c-7337604532e0:2347:1874603274152448:192.168.203.123@tcp:400:kworker.0 00010000:00010000:0.0:1787761244.037031:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761245.062238:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761246.085477:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761247.108923:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761248.133238:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761249.092911:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274152576:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761249.099864:0:2346:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787761249.099893:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274152704:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761249.104943:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:94b87323-e84d-4c2a-846c-789bad072fad:2346:1874603274152576:192.168.203.123@tcp:400:kworker.0 00000100:00100000:0.0:1787761249.115921:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274152704:192.168.203.123@tcp:400:kworker.0 00010000:00010000:3.0:1787761249.160578:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787761250.181019:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761251.206667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1787761251.550751:0:9028:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:1.0:1787761252.229718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761253.252778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761254.151450:0:2337:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.123@tcp (source: 12345-192.168.203.123@tcp) 00000100:00100000:1.0:1787761254.151973:0:5428:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874603295165568 00000100:00100000:1.0:1787761254.151986:0:5428:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874603295165568 00000100:00100000:1.0:1787761254.152048:0:5428:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.123@tcp, seq: 3 00000100:00100000:1.0:1787761254.152057:0:5428:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9ac451f84f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:94b87323-e84d-4c2a-846c-789bad072fad+3:11673:x1874603295165568:12345-192.168.203.123@tcp:104: 00010000:00010000:1.0:1787761254.152074:0:5428:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787761254.152192:0:5428:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1787761254.152198:0:5428:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9ac451f84f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:94b87323-e84d-4c2a-846c-789bad072fad+3:11673:x1874603295165568:12345-192.168.203.123@tcp:104: Request processed in 143us (771us total) trans 0 rc 0/0 00000100:00100000:1.0:1787761254.152206:0:5428:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.123@tcp, seq: 3 00010000:00010000:2.0:1787761254.152248:0:5439:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787761254.152260:0:5439:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761254.152270:0:5439:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9ac4468fec00 already unused, calling callback (ffffffffc12c2c20) 10000000:00010000:2.0:1787761254.152274:0:5439:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761254.152284:0:5439:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787761254.152291:0:5439:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787761254.152297:0:5439:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1787761254.152471:0:5439:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761254.152479:0:5439:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787761254.152512:0:5439:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761254.152598:0:5439:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.123@tcp lock: ffff9ac4468fec00/0x88466b5949dc2eb7 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2dffe29982e246af expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761254.152618:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761254.152620:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761254.152622:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761254.152633:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1787761254.152665:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274152832:192.168.203.123@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787761254.157874:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:94b87323-e84d-4c2a-846c-789bad072fad:2348:1874603274152832:192.168.203.123@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787761254.213452:0:2345:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787761254.213473:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274152960:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761254.215965:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:522fbd36-e7cb-44ce-913c-7337604532e0:2345:1874603274152960:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761254.283058:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761255.300639:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1787761255.364654:0:5442:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1787761255.364662:0:5442:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff9ac4523c1000 from 7 10000000:01000000:3.0:1787761255.364666:0:5442:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1787761255.364716:0:5442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761255.364728:0:5442:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761255.364739:0:5442:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761255.364763:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274153088:192.168.203.123@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1787761255.364865:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4478440c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761255.367967:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274153088:192.168.203.123@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1787761255.367995:0:5442:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e246bd expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761255.368008:0:5442:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2dffe29982e246bd expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761255.368079:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274153216:192.168.203.123@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1787761255.368173:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4478440c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761255.372298:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274153216:192.168.203.123@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1787761255.372355:0:5442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274153344:192.168.203.123@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1787761255.372489:0:5442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4478440c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761255.375905:0:5442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:94b87323-e84d-4c2a-846c-789bad072fad:5442:1874603274153344:192.168.203.123@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1787761255.377103:0:9300:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:3.0:1787761255.377285:0:5442:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:3.0:1787761255.377363:0:5442:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.123@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1787761255.377374:0:5442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e246bd expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761255.377385:0:5442:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.123@tcp lock: ffff9ac4414a8400/0x88466b5949dc2ebe lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2dffe29982e246bd expref: -99 pid: 5442 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787761256.325224:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761257.349610:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1787761258.202754:0:9738:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanity-compr: finish setup 12:20:57 (1787761257) === 00010000:00010000:3.0:1787761258.373175:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1787761258.874839:0:9833:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761258.874860:0:9833:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.874877:0:9833:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1787761258.874891:0:9833:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:3.0:1787761258.874899:0:9833:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761258.874906:0:9833:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761258.874909:0:9833:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:3.0:1787761258.874916:0:9833:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01d03be0 00010000:00010000:3.0:1787761258.874925:0:9833:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1787761258.874933:0:9833:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1787761258.874947:0:9833:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08), inode mode 41ed mask 1 00000080:00200000:3.0:1787761258.874958:0:9833:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffff9ac458524b08), flags=1536 00000080:00200000:3.0:1787761258.874962:0:9833:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761258.874968:0:9833:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.874974:0:9833:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1787761258.874983:0:9833:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1787761258.874990:0:9833:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761258.874994:0:9833:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761258.874995:0:9833:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:3.0:1787761258.874998:0:9833:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01d03ce0 00010000:00010000:3.0:1787761258.875000:0:9833:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1787761258.875005:0:9833:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787761258.875010:0:9833:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08), inode mode 41ed mask 3 00000080:00200000:3.0:1787761258.875013:0:9833:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761258.875021:0:9833:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.875025:0:9833:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787761258.875030:0:9833:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787761258.875034:0:9833:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761258.875035:0:9833:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761258.875037:0:9833:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ea2 00000080:00010000:3.0:1787761258.875039:0:9833:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ea2 from it ffffb76d01d03dc8 00010000:00010000:3.0:1787761258.875040:0:9833:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787761258.875045:0:9833:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787761258.875049:0:9833:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08), inode mode 41ed mask 3 00000080:00200000:3.0:1787761258.875052:0:9833:0:(namei.c:2239:do_mkdir()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffff9ac458524b08) 00800000:00000004:3.0:1787761258.875066:0:9833:0:(obd_class.h:1057:obd_statfs_async()) lustre-MDT0000-mdc-ffff9ac4523c1000: age 214, max_age 0 00000100:00100000:2.0:1787761258.875136:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274153472:192.168.203.123@tcp:41:mkdir.0 00000100:00100000:3.0:1787761258.875136:0:9833:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c51680 pname:cluuid:pid:xid:nid:opc:job mkdir:522fbd36-e7cb-44ce-913c-7337604532e0:9833:1874603274153600:192.168.203.123@tcp:700:mkdir.0 00000100:00100000:3.0:1787761258.875231:0:9833:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787761258.876740:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274153472:192.168.203.123@tcp:41:mkdir.0 00000100:00100000:3.0:1787761258.876740:0:9833:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c51680 pname:cluuid:pid:xid:nid:opc:job mkdir:522fbd36-e7cb-44ce-913c-7337604532e0:9833:1874603274153600:192.168.203.123@tcp:700:mkdir.0 00010000:00010000:3.0:1787761258.876804:0:9833:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787761258.876866:0:9833:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787761258.876885:0:9833:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787761258.876888:0:9833:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ea2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24600 expref: -99 pid: 5467 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000020:00100000:3.0:1787761258.876919:0:9833:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9ac4523c1000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787761258.876928:0:9833:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9ac451c51680 x1874603274153856/t0(0) o36->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 560/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787761258.876947:0:9833:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c51680 pname:cluuid:pid:xid:nid:opc:job mkdir:522fbd36-e7cb-44ce-913c-7337604532e0:9833:1874603274153856:192.168.203.123@tcp:36:mkdir.0 00000100:00100000:3.0:1787761258.877010:0:9833:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761258.878901:0:9833:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c51680 pname:cluuid:pid:xid:nid:opc:job mkdir:522fbd36-e7cb-44ce-913c-7337604532e0:9833:1874603274153856:192.168.203.123@tcp:36:mkdir.0 00000080:00200000:3.0:1787761258.878967:0:9833:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200000401:0x1:0x0](ffff9ac458579988) 00000080:00200000:3.0:1787761258.878975:0:9833:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000401:0x1:0x0], updating i_size 4096 00000080:00200008:3.0:1787761258.883159:0:9834:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787761258.883172:0:9834:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761258.883194:0:9834:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.883250:0:9834:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 9834 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.883258:0:9834:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9834 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.883266:0:9834:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9834 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761258.883292:0:9834:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c521c0 pname:cluuid:pid:xid:nid:opc:job chmod:522fbd36-e7cb-44ce-913c-7337604532e0:9834:1874603274153984:192.168.203.123@tcp:101:chmod.0 00000100:00100000:3.0:1787761258.883379:0:9834:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761258.885089:0:9834:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c521c0 pname:cluuid:pid:xid:nid:opc:job chmod:522fbd36-e7cb-44ce-913c-7337604532e0:9834:1874603274153984:192.168.203.123@tcp:101:chmod.0 00010000:00010000:3.0:1787761258.885119:0:9834:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885130:0:9834:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885141:0:9834:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885152:0:9834:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885164:0:9834:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885173:0:9834:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885180:0:9834:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787761258.885191:0:9834:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9ac451c521c0 x1874603274153984/t0(0) o101->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 576/688 e 0 to 0 dl 1787761274 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chmod.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787761258.885205:0:9834:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885216:0:9834:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9ac448a01000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787761258.885232:0:9834:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787761258.885256:0:9834:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787761258.885262:0:9834:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ec5 00000080:00010000:3.0:1787761258.885268:0:9834:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761258.885271:0:9834:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ec5 from it ffffb76d01d0bcc0 00010000:00010000:3.0:1787761258.885274:0:9834:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.885283:0:9834:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1787761258.885355:0:9834:0:(llite_lib.c:2430:ll_setattr_raw()) START file /:[0x200000007:0x1:0x0](ffff9ac458524b08) current size 4096, valid attrs 41, mode 41ed, uid 1139793920, gid -25916, new size 0, atime 1787761258.000000000, mtime 1787761258.000000000, ctime 1787761258.000000000 00010000:00010000:3.0:1787761258.885376:0:9834:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761258.885425:0:9834:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761258.885432:0:9834:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787761258.885435:0:9834:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ec5 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24703 expref: -99 pid: 9834 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1787761258.885455:0:9834:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9ac4523c1000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787761258.885461:0:9834:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9ac451c521c0 x1874603274154240/t0(0) o36->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787761258.885482:0:9834:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c521c0 pname:cluuid:pid:xid:nid:opc:job chmod:522fbd36-e7cb-44ce-913c-7337604532e0:9834:1874603274154240:192.168.203.123@tcp:36:chmod.0 00000100:00100000:3.0:1787761258.885578:0:9834:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761258.887118:0:9834:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c521c0 pname:cluuid:pid:xid:nid:opc:job chmod:522fbd36-e7cb-44ce-913c-7337604532e0:9834:1874603274154240:192.168.203.123@tcp:36:chmod.0 00000080:00200000:3.0:1787761258.887160:0:9834:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00200008:3.0:1787761258.887205:0:9834:0:(llite_lib.c:2689:ll_setattr_raw()) COMPLETED file /:[0x200000007:0x1:0x0](ffff9ac458524b08) current size 4096, valid attrs 10000041, mode 41ed, uid 1139793920, gid -25916, new size 0, atime 1787761258.000000000, mtime 1787761258.000000000, ctime 1787761258.000000000, rc 0 00000080:00200000:3.0:1787761258.897495:0:9835:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761258.897520:0:9835:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.897612:0:9835:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 9835 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.897624:0:9835:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9835 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.897636:0:9835:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9835 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761258.897656:0:9835:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c51a40 pname:cluuid:pid:xid:nid:opc:job chown:522fbd36-e7cb-44ce-913c-7337604532e0:9835:1874603274154368:192.168.203.123@tcp:101:chown.0 00000100:00100000:3.0:1787761258.897735:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761258.900067:0:9835:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c51a40 pname:cluuid:pid:xid:nid:opc:job chown:522fbd36-e7cb-44ce-913c-7337604532e0:9835:1874603274154368:192.168.203.123@tcp:101:chown.0 00010000:00010000:3.0:1787761258.900089:0:9835:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900113:0:9835:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900126:0:9835:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900139:0:9835:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900162:0:9835:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900170:0:9835:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900179:0:9835:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787761258.900191:0:9835:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9ac451c51a40 x1874603274154368/t0(0) o101->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 576/688 e 0 to 0 dl 1787761274 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chown.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787761258.900205:0:9835:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900216:0:9835:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9ac448a01000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787761258.900222:0:9835:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787761258.900243:0:9835:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787761258.900248:0:9835:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ecc 00000080:00010000:3.0:1787761258.900253:0:9835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761258.900257:0:9835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ecc from it ffffb76d01d13b58 00010000:00010000:3.0:1787761258.900260:0:9835:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900270:0:9835:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787761258.900304:0:9835:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08), inode mode 41ed mask 1 00000080:00200000:3.0:1787761258.900319:0:9835:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffff9ac458524b08), flags=8 00000080:00200000:3.0:1787761258.900330:0:9835:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffff9ac458524b08), intent=getattr 00000002:00010000:3.0:1787761258.900346:0:9835:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d0_runas_test,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1787761258.900377:0:9835:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 9835 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900385:0:9835:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9835 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.900393:0:9835:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9835 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787761258.900409:0:9835:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c51a40 pname:cluuid:pid:xid:nid:opc:job chown:522fbd36-e7cb-44ce-913c-7337604532e0:9835:1874603274154496:192.168.203.123@tcp:101:chown.0 00000100:00100000:3.0:1787761258.900462:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787761258.902134:0:9835:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c51a40 pname:cluuid:pid:xid:nid:opc:job chown:522fbd36-e7cb-44ce-913c-7337604532e0:9835:1874603274154496:192.168.203.123@tcp:101:chown.0 00010000:00010000:3.0:1787761258.902150:0:9835:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902168:0:9835:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902176:0:9835:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902184:0:9835:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902207:0:9835:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902217:0:9835:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902224:0:9835:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902231:0:9835:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787761258.902243:0:9835:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9ac451c51a40 x1874603274154496/t0(0) o101->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 584/584 e 0 to 0 dl 1787761274 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chown.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787761258.902253:0:9835:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902262:0:9835:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9ac448a01000 type 13 mode 4 res 8589935617/1 (0 0) 00000080:00200000:3.0:1787761258.902266:0:9835:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1787761258.902281:0:9835:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000401:0x1:0x0], updating i_size 4096 00000080:00200000:3.0:1787761258.902284:0:9835:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x1:0x0](ffff9ac458579988): rc = 0 00000080:00010000:3.0:1787761258.902288:0:9835:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1:0x0](ffff9ac458579988) for lock 0x88466b5949dc2ed3 00000080:00010000:3.0:1787761258.902298:0:9835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1:0x0](ffff9ac458579988) 00000080:00010000:3.0:1787761258.902300:0:9835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ed3 from it ffffb76d01d13b20 00010000:00010000:3.0:1787761258.902303:0:9835:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787761258.902322:0:9835:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1787761258.902349:0:9835:0:(file.c:6313:ll_getattr_dentry()) START file d0_runas_test:[0x200000401:0x1:0x0](ffff9ac458579988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787761258.902355:0:9835:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x1:0x0](ffff9ac458579988),name=d0_runas_test 00000002:00010000:3.0:1787761258.902367:0:9835:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000401:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.902375:0:9835:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761258.902384:0:9835:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787761258.902390:0:9835:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761258.902394:0:9835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1:0x0](ffff9ac458579988) 00000080:00010000:3.0:1787761258.902397:0:9835:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1:0x0](ffff9ac458579988) for lock 0x88466b5949dc2ed3 00000080:00010000:3.0:1787761258.902400:0:9835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ed3 from it ffffb76d01d13cc0 00010000:00010000:3.0:1787761258.902403:0:9835:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761258.902410:0:9835:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787761258.902456:0:9835:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08),name=/ 00000002:00010000:3.0:1787761258.902462:0:9835:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787761258.902468:0:9835:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761258.902486:0:9835:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787761258.902492:0:9835:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787761258.902496:0:9835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) 00000080:00010000:3.0:1787761258.902498:0:9835:0:(llite_internal.h:1991:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9ac458524b08) for lock 0x88466b5949dc2ecc 00000080:00010000:3.0:1787761258.902502:0:9835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x88466b5949dc2ecc from it ffffb76d01d13bf8 00010000:00010000:3.0:1787761258.902505:0:9835:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787761258.902512:0:9835:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787761258.902520:0:9835:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9ac458524b08), inode mode 41ed mask 1 00000080:00200000:3.0:1787761258.902525:0:9835:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d0_runas_test, flags=9 00000080:00200008:3.0:1787761258.902652:0:9835:0:(llite_lib.c:2430:ll_setattr_raw()) START file d0_runas_test:[0x200000401:0x1:0x0](ffff9ac458579988) current size 4096, valid attrs 46, mode ffff, uid 500, gid 500, new size 18446632766476902400, atime 1787761258.000000000, mtime 1787761258.000000000, ctime 1787761258.000000000 00010000:00010000:3.0:1787761258.902670:0:9835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787761258.902705:0:9835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787761258.902712:0:9835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787761258.902715:0:9835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8200/0x88466b5949dc2ed3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24734 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1787761258.902746:0:9835:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9ac4523c1000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787761258.902753:0:9835:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9ac451c51a40 x1874603274154752/t0(0) o36->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787761258.902770:0:9835:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c51a40 pname:cluuid:pid:xid:nid:opc:job chown:522fbd36-e7cb-44ce-913c-7337604532e0:9835:1874603274154752:192.168.203.123@tcp:36:chown.0 00000100:00100000:3.0:1787761258.902819:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 11 seconds 00000100:00100000:0.0:1787761259.332784:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274155008:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761259.332796:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274154880:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761259.396675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761260.420727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761261.444676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761262.468732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761263.492671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761264.452747:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274155264:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761264.452755:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274155136:192.168.203.123@tcp:400:kworker.0 00010000:00010000:3.0:1787761264.516703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761265.540629:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761266.564765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787761267.588683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787761268.612756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761269.572706:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274155392:192.168.203.123@tcp:400:kworker.0 00000100:00100000:2.0:1787761269.572707:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274155520:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761269.636657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761270.084639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 1 seconds 00010000:00010000:3.0:1787761270.660705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761271.108591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 1 seconds 00010000:00010000:2.0:1787761271.684676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761272.132655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 1 seconds 00010000:00010000:3.0:1787761272.708701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761273.156624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 1 seconds 00010000:00010000:2.0:1787761273.732636:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1787761274.180649:0:9835:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761258/real 1787761258] req@ffff9ac451c51a40 x1874603274154752/t0(0) o36->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 488/512 e 0 to 1 dl 1787761274 ref 2 fl Rpc:XQr/200/ffffffff rc 0/-1 job:'chown.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761274.187587:0:9835:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1787761274.187601:0:9835:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff9ac4523c1000: Connection to lustre-MDT0000 (at 192.168.203.123@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1787761274.191651:0:9835:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1787761274.191695:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761274.191751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761274.191756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761274.191759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761274.191768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 210 00000100:00080000:0.0:1787761274.191779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761274.191786:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274155648:192.168.203.123@tcp:400:kworker.0 00000008:00100000:0.0:1787761274.191789:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761274.191811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274155776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761274.191859:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274155776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761274.756665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761275.204637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00000400:1.0:1787761275.204650:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761259/real 1787761259] req@ffff9ac451f870c0 x1874603274154880/t0(0) o400->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 224/224 e 0 to 1 dl 1787761275 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761275.212345:0:2345:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9ac451f870c0 x1874603274154880/t0(0) o400->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 224/224 e 0 to 1 dl 1787761275 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761275.212356:0:2345:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1787761275.212359:0:2345:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.123@tcp: Connection to MGS (at 192.168.203.123@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1787761275.215376:0:2345:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.123@tcp for MGC192.168.203.123@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1787761275.215378:0:2345:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1787761275.215382:0:2345:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9ac451f870c0 x1874603274154880/t0(0) o400->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 224/224 e 0 to 1 dl 1787761275 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761275.215391:0:2345:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9ac451f87c00 x1874603274155136/t0(0) o400->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 224/224 e 0 to 0 dl 1787761280 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761275.215416:0:2345:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9ac4523bcb40 x1874603274155392/t0(0) o400->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 224/224 e 0 to 0 dl 1787761285 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761275.215431:0:2345:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9ac451f84b40 x1874603274155648/t0(0) o400->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 224/224 e 0 to 0 dl 1787761290 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1787761275.215437:0:2345:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1787761275.215439:0:2345:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1787761275.215440:0:2345:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1787761275.215441:0:2345:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:1.0:1787761275.215470:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274155648:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761275.215478:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274155136:192.168.203.123@tcp:400:kworker.0 00000100:00100000:1.0:1787761275.215484:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:94b87323-e84d-4c2a-846c-789bad072fad:2345:1874603274154880:192.168.203.123@tcp:400:kworker.0 00000100:00100000:2.0:1787761275.215490:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:94b87323-e84d-4c2a-846c-789bad072fad:2347:1874603274155392:192.168.203.123@tcp:400:kworker.0 00000100:00080000:0.0:1787761275.215499:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761275.215506:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761275.215509:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761275.215516:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 210 00000100:00080000:0.0:1787761275.215562:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761275.215589:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274155904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761275.215609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761275.215612:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1787761275.215642:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274155904:192.168.203.123@tcp:250:kworker.0 00000100:00000400:0.0:1787761275.716601:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761259/real 1787761259] req@ffff9ac451f843c0 x1874603274155008/t0(0) o400->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/224 e 0 to 1 dl 1787761275 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761275.716641:0:2346:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9ac451f843c0 x1874603274155008/t0(0) o400->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/224 e 0 to 1 dl 1787761275 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761275.716647:0:2346:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9ac4454bf800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:0.0:1787761275.716660:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274155008:192.168.203.123@tcp:400:kworker.0 00010000:00010000:2.0:1787761275.780603:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761276.228610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761276.804672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761277.252641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787761277.828643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761278.276659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761278.852710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761279.236630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761279.236638:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761279.236641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761279.236643:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787761279.300624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761279.876730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761280.260611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761280.260623:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761280.260626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761280.260629:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787761280.324633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00000400:1.0:1787761280.836646:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761264/real 1787761264] req@ffff9ac451f86940 x1874603274155264/t0(0) o400->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/224 e 0 to 1 dl 1787761280 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761280.849278:0:2346:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9ac451f86940 x1874603274155264/t0(0) o400->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/224 e 0 to 1 dl 1787761280 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761280.849288:0:2346:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9ac4454bf800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:1.0:1787761280.849318:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:522fbd36-e7cb-44ce-913c-7337604532e0:2346:1874603274155264:192.168.203.123@tcp:400:kworker.0 00010000:00010000:3.0:1787761280.900879:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761281.348609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761281.924671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761282.372662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761282.949685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761283.396628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761283.972746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761284.356647:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761274/real 1787761274] req@ffff9ac451f84780 x1874603274155776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761284 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761284.356682:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274155776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761284 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761284.356698:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761284.356718:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761284.356721:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274155776:192.168.203.123@tcp:38:kworker.0 00000100:00080000:0.0:1787761284.356763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761284.356769:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761284.356777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761284.356780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761284.356782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761284.356788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 324 00000100:00080000:0.0:1787761284.356798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1787761284.356803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761284.356828:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761284.356851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274156032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761284.356900:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156032:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787761284.420655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761284.997643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761285.380636:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761275/real 1787761275] req@ffff9ac451f84000 x1874603274155904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761285 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761285.380692:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274155904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761285 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761285.380710:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761285.380733:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761285.380738:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274155904:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787761285.380791:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761285.380797:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761285.380801:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761285.380814:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 325 00000100:00080000:3.0:1787761285.380824:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1787761285.380831:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761285.380860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274156160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761285.380884:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761285.380889:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1787761285.380929:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761285.444680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00000400:3.0:1787761285.956663:0:2348:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761269/real 1787761269] req@ffff9ac4523be580 x1874603274155520/t0(0) o400->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/224 e 0 to 1 dl 1787761285 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761285.964997:0:2348:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9ac4523be580 x1874603274155520/t0(0) o400->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 224/224 e 0 to 1 dl 1787761285 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787761285.965007:0:2348:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9ac4454bf800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:3.0:1787761285.965021:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:522fbd36-e7cb-44ce-913c-7337604532e0:2348:1874603274155520:192.168.203.123@tcp:400:kworker.0 00010000:00010000:1.0:1787761286.020710:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761286.468603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761287.044677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761287.492622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761288.068713:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761288.516607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761289.092906:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761289.412648:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761289.412659:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761289.412663:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761289.412666:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787761289.540627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761290.116747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761290.436630:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761290.436639:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761290.436642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761290.436645:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787761290.564713:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761291.140712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761291.588640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761292.164662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761292.612620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761293.188718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761293.636618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761294.212752:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761294.660592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761295.236663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761295.684598:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761295.684606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761295.684606:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787761295.684610:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761295.684615:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1787761296.260673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761296.708601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761297.284708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761297.732606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761298.308674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761298.756609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761299.332682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761299.652592:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761284/real 1787761284] req@ffff9ac451f843c0 x1874603274156032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761299 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761299.652621:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274156032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761299 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761299.652637:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761299.652656:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761299.652660:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156032:192.168.203.123@tcp:38:kworker.0 00000100:00080000:0.0:1787761299.652704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761299.652710:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761299.652712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761299.652714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761299.652717:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761299.652723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 334 00000100:00080000:0.0:1787761299.652732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1787761299.652737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761299.652746:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761299.652773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274156288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761299.652842:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156288:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787761299.780600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761300.356650:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761300.676620:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761285/real 1787761285] req@ffff9ac451c50780 x1874603274156160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761300 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761300.676652:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274156160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761300 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761300.676668:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761300.676687:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761300.676707:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156160:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787761300.676737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761300.676744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761300.676747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761300.676755:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 335 00000100:00080000:0.0:1787761300.676766:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1787761300.676773:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761300.676814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274156416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761300.676837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761300.676841:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1787761300.676866:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761300.804600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761301.380670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761301.828623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761302.404671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761302.852597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761303.428639:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761303.876617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761304.452651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761304.708607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761304.708615:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761304.708618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761304.708621:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787761304.900612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761305.476644:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761305.732596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761305.732603:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761305.732606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761305.732609:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787761305.924587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761306.500653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761306.948580:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761307.524684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761307.972618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761308.548746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761308.996623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761309.572671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761310.020611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761310.596711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761311.044618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761311.044632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761311.044640:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787761311.044643:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761311.044645:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1787761311.620690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761312.068610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761312.644660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761313.092632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761313.668653:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761314.116596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761314.692684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761315.140587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761315.716737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761316.164603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761316.164621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761316.164628:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761316.164631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761316.164647:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1787761316.740644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761317.188606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761317.764640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761318.212597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761318.788650:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1787761318.980612:0:2336:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.123@tcp (192.168.203.123:988) timed out; the network or node may be down. 00000100:00100000:2.0:1787761319.044645:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761299/real 1787761299] req@ffff9ac451f861c0 x1874603274156288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761319 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761319.044673:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274156288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761319 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761319.044703:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761319.044723:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761319.044726:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156288:192.168.203.123@tcp:38:kworker.0 00000100:00080000:0.0:1787761319.044776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761319.044782:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787761319.044784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761319.044786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761319.044789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761319.044795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 349 00000100:00080000:0.0:1787761319.044803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787761319.044809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761319.044818:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761319.044845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274156544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761319.044893:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156544:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787761319.236621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761319.813613:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761320.068603:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787761300/real 1787761300] req@ffff9ac451f85680 x1874603274156416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761320 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761320.068632:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274156416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761320 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761320.068649:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761320.068677:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761320.068681:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156416:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787761320.068735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761320.068740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761320.068743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761320.068748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 350 00000100:00080000:0.0:1787761320.068759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787761320.068765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761320.068793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274156672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761320.068814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761320.068817:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1787761320.068844:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761320.260622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761320.836705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761321.284598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761321.476934:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.123:988 00000400:00000100:1.0:1787761321.476973:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761321.476991:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761321.477033:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761321.477086:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761320/real 1787761321] req@ffff9ac451f843c0 x1874603274156672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761321.477134:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274156672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761321.477145:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761321.477151:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761321.477156:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761321.477165:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761319/real 1787761321] req@ffff9ac451f861c0 x1874603274156544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761321.477195:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274156544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761321.477204:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761321.477208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761321.477212:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761321.861615:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761322.308607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761322.884721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761323.332593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761323.908686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761324.100598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761324.100607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761324.100611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761324.100621:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 369 00000100:00080000:0.0:1787761324.100631:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761324.100650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761324.100687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274156800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761324.100710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761324.100713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761324.100714:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761324.100717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 368 00000100:00080000:0.0:1787761324.100720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761324.100723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761324.100732:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761324.100741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274156928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761324.100765:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156800:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761324.100914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156928:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787761324.356635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761324.548820:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.123:988 00000400:00000100:1.0:1787761324.548870:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761324.548889:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761324.548925:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761324.549007:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761324/real 1787761324] req@ffff9ac451f84780 x1874603274156928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761324.549035:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274156928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761324.549048:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761324.549053:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761324.549077:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274156928:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787761324.549087:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761324/real 1787761324] req@ffff9ac451f84000 x1874603274156800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761324.549102:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274156800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761324.549110:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761324.549113:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761324.549117:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274156800:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787761324.932698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761325.380612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761325.956646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761326.404648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761326.981651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761327.428594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761328.005606:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761328.452600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761329.028717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761329.476627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761329.476638:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761329.476644:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761329.476648:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761329.476656:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 373 00000100:00080000:3.0:1787761329.476666:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761329.476671:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761329.476701:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274157056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761329.476732:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761329.476735:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761329.476736:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761329.476739:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 373 00000100:00080000:3.0:1787761329.476743:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761329.476746:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761329.476754:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761329.476764:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274157184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761329.476773:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761329.476879:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157184:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761330.052654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761330.500636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761330.692867:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.123:988 00000400:00000100:1.0:1787761330.692907:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761330.692923:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761330.692967:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761330.693086:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761329/real 1787761330] req@ffff9ac451c52580 x1874603274157184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761330.693126:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274157184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761330.693140:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761330.693145:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761330.693150:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157184:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787761330.693158:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761329/real 1787761330] req@ffff9ac451c50f00 x1874603274157056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761330.693170:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274157056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761330.693206:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761330.693211:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761330.693216:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157056:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787761331.076716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761331.524608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761332.100693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761332.548622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761333.124712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761333.572590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761334.148657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761334.596611:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761334.596619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761334.596622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761334.596629:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761334.596640:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 379 00000100:00080000:0.0:1787761334.596650:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761334.596658:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761334.596691:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274157312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761334.596726:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761334.596730:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761334.596732:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761334.596737:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 379 00000100:00080000:0.0:1787761334.596742:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761334.596747:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761334.596757:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761334.596766:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157312:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761334.596773:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274157440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761334.596854:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157440:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761335.172659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761335.620633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761336.196708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761336.644621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761336.836874:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.123:988 00000400:00000100:1.0:1787761336.836925:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761336.836941:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761336.836976:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761336.837023:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761334/real 1787761336] req@ffff9ac451f870c0 x1874603274157440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761336.837051:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274157440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761336.837062:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761336.837067:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761336.837073:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157440:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787761336.837081:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761334/real 1787761336] req@ffff9ac451f86940 x1874603274157312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761336.837095:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274157312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761336.837103:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761336.837107:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761336.837110:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157312:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787761337.220676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761337.668625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761338.244649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761338.692648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761339.268641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761339.716599:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761339.716602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761339.716611:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761339.716616:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761339.716626:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 384 00000100:00080000:3.0:1787761339.716636:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761339.716643:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761339.716679:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274157568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761339.716702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761339.716706:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761339.716719:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761339.716725:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 384 00000100:00080000:3.0:1787761339.716730:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761339.716738:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761339.716739:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157568:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787761339.716749:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761339.716765:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274157696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761339.716803:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761339.716835:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761339/real 1787761339] req@ffff9ac451c50f00 x1874603274157568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761339.716848:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274157568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761339.716856:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761339.716859:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761339.716861:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157568:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761339.716883:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157696:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761339.716906:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761339.716919:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761339/real 1787761339] req@ffff9ac451c52580 x1874603274157696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761339.716933:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274157696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761339.716942:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761339.716945:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761339.716949:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157696:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761340.292644:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761340.740606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761341.316670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761341.764584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761342.340671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761342.788606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761343.364644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761343.812601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761344.388642:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761344.836708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761344.836716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761344.836726:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761344.836732:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761344.836746:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 389 00000100:00080000:3.0:1787761344.836761:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761344.836770:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761344.836821:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274157824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761344.836855:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761344.836859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761344.836861:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761344.836866:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 389 00000100:00080000:3.0:1787761344.836870:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761344.836873:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761344.836886:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761344.836917:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274157952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761344.836930:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157824:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761344.837062:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157952:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761345.412665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761345.860609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761346.052777:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.123:988 00000400:00000100:1.0:1787761346.052833:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761346.052855:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761346.052904:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761346.052954:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761344/real 1787761346] req@ffff9ac451c503c0 x1874603274157952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761346.053010:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274157952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761346.053024:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761346.053030:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761346.053035:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274157952:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787761346.053044:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761344/real 1787761346] req@ffff9ac451c50780 x1874603274157824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761346.053060:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274157824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761346.053069:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761346.053073:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761346.053078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274157824:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787761346.436661:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761346.884589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761347.460644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761347.908600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761348.484644:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761348.932609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761349.508657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761349.956586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761349.956599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761349.956605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761349.956609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761349.956617:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 394 00000100:00080000:2.0:1787761349.956626:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761349.956632:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761349.956667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274158080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761349.956689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761349.956705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761349.956706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761349.956710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 394 00000100:00080000:2.0:1787761349.956713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761349.956717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761349.956725:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761349.956735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274158208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761349.956746:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158080:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761349.956810:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761349.956847:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761349/real 1787761349] req@ffff9ac4523bcb40 x1874603274158080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761349.956869:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274158080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761349.956877:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761349.956880:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761349.956883:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158080:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761349.956901:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158208:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761349.956918:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761349.956926:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761349/real 1787761349] req@ffff9ac4523bcf00 x1874603274158208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761349.956943:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274158208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761349.956949:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761349.956951:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761349.956953:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158208:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761350.532637:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761350.980594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761351.556710:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761352.004597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761352.580654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761353.028597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761353.604653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761354.052611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761354.628643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761355.076601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761355.076615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761355.076623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761355.076627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761355.076638:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 399 00000100:00080000:0.0:1787761355.076649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761355.076657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761355.076690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274158336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761355.076714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761355.076716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761355.076718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761355.076722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 399 00000100:00080000:0.0:1787761355.076726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761355.076730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761355.076738:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761355.076743:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761355.076750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274158464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761355.076812:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761355.076845:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761355/real 1787761355] req@ffff9ac451f87c00 x1874603274158336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761355.076858:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274158336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761355.076866:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761355.076870:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761355.076873:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761355.076891:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158464:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761355.076908:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761355.076916:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761355/real 1787761355] req@ffff9ac451f84b40 x1874603274158464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761355.076933:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274158464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761355.076946:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761355.076949:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761355.076951:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158464:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761355.652667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761356.100601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761356.676643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761357.124614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761357.700677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761358.148605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761358.724666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761359.172602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761359.748655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761360.196610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761360.196659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761360.196671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761360.196676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761360.196689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 404 00000100:00080000:0.0:1787761360.196698:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761360.196705:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761360.196741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274158592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761360.196765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761360.196768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761360.196770:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761360.196773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 404 00000100:00080000:0.0:1787761360.196776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761360.196778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761360.196786:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761360.196796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274158720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761360.196822:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158592:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761360.196890:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761360.196942:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761360/real 1787761360] req@ffff9ac451f84b40 x1874603274158592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761360.196963:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274158592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761360.196972:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761360.196975:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761360.196978:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158592:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761360.196985:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158720:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761360.197003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761360.197012:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761360/real 1787761360] req@ffff9ac451f87c00 x1874603274158720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761360.197021:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274158720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761360.197026:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761360.197029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761360.197031:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158720:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761360.772635:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761361.220586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761361.796678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761362.244603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761362.820658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761363.268602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761363.844665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761364.292600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761364.868641:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761365.316604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761365.316609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761365.316617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761365.316622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761365.316631:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 410 00000100:00080000:2.0:1787761365.316643:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761365.316650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761365.316691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274158848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761365.316717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761365.316720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761365.316722:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761365.316725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 410 00000100:00080000:2.0:1787761365.316728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761365.316731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761365.316749:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761365.316751:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158848:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761365.316760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274158976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761365.316866:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158976:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761365.892671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761366.340617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761366.916682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761367.364598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761367.556788:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.123:988 00000400:00000100:1.0:1787761367.556834:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761367.556852:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761367.556893:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787761367.556947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761365/real 1787761367] req@ffff9ac4523be1c0 x1874603274158976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761367.556983:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274158976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761367.556993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761367.556998:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761367.557002:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274158976:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787761367.557011:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761365/real 1787761367] req@ffff9ac4523bf0c0 x1874603274158848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761367.557027:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274158848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761367.557036:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761367.557040:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761367.557044:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274158848:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787761367.940655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761368.388599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761368.964645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761369.412600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761369.988632:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761370.436598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761370.436609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761370.436618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761370.436622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761370.436632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 415 00000100:00080000:2.0:1787761370.436644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761370.436651:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761370.436691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274159104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761370.436723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761370.436727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761370.436728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761370.436731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 415 00000100:00080000:2.0:1787761370.436734:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761370.436737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761370.436747:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761370.436758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274159232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761370.436788:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159104:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761370.436860:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761370.436903:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761370/real 1787761370] req@ffff9ac4523bf0c0 x1874603274159104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761370.436932:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274159104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761370.436943:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761370.436948:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761370.436953:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761370.436973:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274159232:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761370.436999:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761370.437013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761370/real 1787761370] req@ffff9ac4523be1c0 x1874603274159232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761370.437025:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274159232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761370.437033:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761370.437038:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761370.437041:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274159232:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761371.012723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761371.460583:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761372.036695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761372.484593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761373.060662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761373.508590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761374.084667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761374.532593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761375.108657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761375.556596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761375.556616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761375.556622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761375.556626:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761375.556635:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 420 00000100:00080000:0.0:1787761375.556644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761375.556650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761375.556682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274159360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761375.556707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761375.556711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761375.556713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761375.556718:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 420 00000100:00080000:0.0:1787761375.556722:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761375.556739:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761375.556742:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159360:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787761375.556749:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761375.556761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274159488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761375.556806:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761375.556841:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761375/real 1787761375] req@ffff9ac451f86940 x1874603274159360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761375.556853:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274159360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761375.556861:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761375.556864:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761375.556867:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159360:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761375.556884:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274159488:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761375.556900:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761375.556908:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761375/real 1787761375] req@ffff9ac451f870c0 x1874603274159488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761375.556923:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274159488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761375.556930:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761375.556932:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761375.556934:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274159488:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761376.132644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761376.580589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761377.156643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761377.604600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761378.180685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761378.628617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761379.204669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761379.652614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761380.228681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761380.676589:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761380.676598:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761380.676601:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787761380.676605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761380.676611:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 425 00000100:00080000:3.0:1787761380.676621:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761380.676628:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761380.676656:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274159616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761380.676677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761380.676679:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761380.676688:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761380.676692:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 425 00000100:00080000:3.0:1787761380.676695:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761380.676699:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761380.676707:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761380.676720:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274159744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761380.676721:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159616:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761380.676802:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761380.676840:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761380/real 1787761380] req@ffff9ac451c52940 x1874603274159616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761380.676858:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274159616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761380.676875:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761380.676879:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761380.676883:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159616:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761380.676908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274159744:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761380.676931:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761380.676943:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761380/real 1787761380] req@ffff9ac451c50b40 x1874603274159744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761380.676956:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274159744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761380.676977:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761380.676981:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761380.676985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274159744:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761381.252642:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761381.700587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761382.276648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761382.724580:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761383.300706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761383.748590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761384.324643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761384.772590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761385.348652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761385.796595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761385.796595:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761385.796603:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761385.796614:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761385.796626:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 430 00000100:00080000:3.0:1787761385.796635:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761385.796641:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761385.796670:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274159872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761385.796690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761385.796692:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761385.796694:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761385.796697:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 430 00000100:00080000:3.0:1787761385.796699:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761385.796702:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761385.796708:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761385.796716:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274160000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761385.796743:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159872:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761385.796810:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761385.796850:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761385/real 1787761385] req@ffff9ac451c52d00 x1874603274159872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761385.796867:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274159872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761385.796877:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761385.796881:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761385.796885:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274159872:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761385.796911:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160000:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761385.796935:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761385.796948:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761385/real 1787761385] req@ffff9ac451c530c0 x1874603274160000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761385.796962:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274160000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761385.796968:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761385.796970:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761385.796973:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160000:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761386.372645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761386.820595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761387.396666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761387.844593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761388.420710:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761388.868612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761389.444677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761389.892645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761390.468662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761390.916586:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761390.916594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761390.916595:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761390.916601:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761390.916611:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 435 00000100:00080000:3.0:1787761390.916621:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761390.916627:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761390.916659:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274160128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761390.916681:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761390.916684:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761390.916685:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761390.916692:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 435 00000100:00080000:3.0:1787761390.916694:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761390.916697:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761390.916704:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761390.916714:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274160256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761390.916729:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160128:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761390.916785:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761390.916818:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761390/real 1787761390] req@ffff9ac451c530c0 x1874603274160128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761390.916831:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274160128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761390.916839:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761390.916842:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761390.916845:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160128:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761390.916867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160256:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761390.916891:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761390.916899:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761390/real 1787761390] req@ffff9ac451c52d00 x1874603274160256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761390.916908:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274160256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761390.916914:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761390.916916:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761390.916919:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160256:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761391.492715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761391.940597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761392.516680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761392.964607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761393.540670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761393.988595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761394.564656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761395.012600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761395.588668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761396.036607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761396.036616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761396.036622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761396.036626:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761396.036634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 440 00000100:00080000:0.0:1787761396.036644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761396.036650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761396.036680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274160384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761396.036700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761396.036703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761396.036704:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761396.036707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 440 00000100:00080000:0.0:1787761396.036710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761396.036712:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761396.036730:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761396.036740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274160512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761396.036755:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160384:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761396.036813:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761396.036845:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761396/real 1787761396] req@ffff9ac451f84000 x1874603274160384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761396.036865:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274160384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761396.036873:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761396.036876:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761396.036878:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160384:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761396.036896:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160512:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761396.036912:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761396.036920:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761396/real 1787761396] req@ffff9ac451f84780 x1874603274160512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761396.036928:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274160512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761396.036934:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761396.036936:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761396.036938:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160512:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761396.612658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761397.060605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761397.636638:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761398.084614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761398.660702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761399.108613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761399.684680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761400.132618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761400.708671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761401.156605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761401.156612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761401.156620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761401.156624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761401.156634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 445 00000100:00080000:0.0:1787761401.156644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761401.156652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761401.156686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274160640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761401.156711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761401.156714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761401.156716:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761401.156721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 445 00000100:00080000:0.0:1787761401.156725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761401.156730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761401.156741:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761401.156749:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160640:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761401.156756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274160768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761401.156841:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160768:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761401.732656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761402.180645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761402.756664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761403.204646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761403.780668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761404.228728:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787761404.228831:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787761404.228890:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787761404.228909:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787761404.228962:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787761404.229034:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761401/real 1787761404] req@ffff9ac451f843c0 x1874603274160768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761404.229073:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274160768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761404.229115:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761404.229121:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761404.229126:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274160768:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787761404.229134:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761401/real 1787761404] req@ffff9ac451f861c0 x1874603274160640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761404.229149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274160640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761404.229156:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761404.229160:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761404.229162:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160640:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787761404.804701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761405.252664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761405.828688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787761406.276626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761406.276634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761406.276638:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761406.276647:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 451 00000100:00080000:2.0:1787761406.276658:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761406.276664:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761406.276683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787761406.276703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274160896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761406.276731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761406.276735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761406.276737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761406.276742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 451 00000100:00080000:2.0:1787761406.276746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761406.276751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761406.276761:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761406.276771:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761406.276774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274161024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761406.276833:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761406.276868:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761406/real 1787761406] req@ffff9ac4523bcf00 x1874603274160896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761406.276881:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274160896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761406.276888:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761406.276891:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761406.276894:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274160896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761406.276912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161024:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761406.276952:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761406.276967:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761406/real 1787761406] req@ffff9ac4523bcb40 x1874603274161024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761406.276979:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274161024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761406.276987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761406.276991:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761406.276994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161024:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761406.852647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761407.300689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761407.876675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761408.324592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761408.900672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761409.348588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761409.924663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761410.372596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761410.948698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761411.396596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761411.396651:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761411.396670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761411.396673:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761411.396681:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 456 00000100:00080000:3.0:1787761411.396690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761411.396697:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761411.396726:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274161152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761411.396750:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761411.396753:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761411.396755:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761411.396758:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 456 00000100:00080000:3.0:1787761411.396761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761411.396763:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761411.396771:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761411.396779:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274161280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761411.396828:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161152:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761411.396886:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761411.396919:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761411/real 1787761411] req@ffff9ac451c50b40 x1874603274161152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761411.396932:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274161152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761411.396939:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761411.396942:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761411.396945:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161152:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761411.396950:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161280:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761411.396963:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761411.396970:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761411/real 1787761411] req@ffff9ac451c52940 x1874603274161280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761411.396987:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274161280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761411.396993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761411.396995:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761411.396997:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161280:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761411.972685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761412.420611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761412.996689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761413.444604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761414.020660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761414.468612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761415.044656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761415.492596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761416.068737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761416.516597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761416.516600:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761416.516607:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761416.516611:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761416.516619:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 461 00000100:00080000:3.0:1787761416.516629:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761416.516635:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761416.516668:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274161408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761416.516708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761416.516711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761416.516712:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761416.516715:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 461 00000100:00080000:3.0:1787761416.516718:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761416.516721:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761416.516729:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761416.516737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274161536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761416.516787:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161408:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761416.516842:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761416.516876:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761416/real 1787761416] req@ffff9ac451c50780 x1874603274161408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761416.516889:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274161408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761416.516897:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761416.516900:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761416.516903:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161408:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761416.516908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161536:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761416.516922:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761416.516929:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761416/real 1787761416] req@ffff9ac451c503c0 x1874603274161536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761416.516950:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274161536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761416.516956:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761416.516958:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761416.516961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161536:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761417.092758:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761417.540596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761418.116685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761418.564595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761419.140674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761419.588598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761420.164662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761420.612656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761421.188714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761421.636636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761421.636657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761421.636667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761421.636672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761421.636685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 466 00000100:00080000:2.0:1787761421.636698:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761421.636707:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761421.636756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274161664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761421.636810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761421.636815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761421.636816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761421.636821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 466 00000100:00080000:2.0:1787761421.636825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761421.636829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761421.636840:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761421.636852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274161792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761421.636907:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161664:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761421.636982:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761421.637023:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761421/real 1787761421] req@ffff9ac4523be580 x1874603274161664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761421.637037:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274161664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761421.637045:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761421.637049:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761421.637052:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761421.637057:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161792:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761421.637072:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761421.637080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761421/real 1787761421] req@ffff9ac4523bc780 x1874603274161792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761421.637087:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274161792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761421.637105:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761421.637108:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761421.637110:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274161792:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761422.212663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761422.660602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761423.236663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761423.684592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761424.260646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761424.708588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761425.284655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761425.732592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761426.308654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761426.756601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761426.756617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761426.756624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761426.756629:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761426.756640:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 471 00000100:00080000:2.0:1787761426.756650:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761426.756657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761426.756694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274161920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761426.756722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761426.756726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761426.756728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761426.756732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 471 00000100:00080000:2.0:1787761426.756736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761426.756740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761426.756762:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761426.756767:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161920:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761426.756781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274162048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761426.756834:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761426.756877:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761426/real 1787761426] req@ffff9ac4523bd680 x1874603274161920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761426.756891:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274161920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761426.756899:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761426.756902:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761426.756905:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274161920:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761426.756937:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162048:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761426.756957:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761426.756967:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761426/real 1787761426] req@ffff9ac4523bc000 x1874603274162048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761426.756976:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274162048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761426.756992:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761426.756995:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761426.756997:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162048:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761427.332657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761427.780616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761428.356660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761428.804606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761429.380657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761429.828598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761430.404707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761430.852592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761431.428698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761431.876612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761431.876637:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761431.876642:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761431.876646:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761431.876654:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 476 00000100:00080000:3.0:1787761431.876664:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761431.876670:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761431.876701:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274162176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761431.876723:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761431.876726:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761431.876727:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761431.876730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 476 00000100:00080000:3.0:1787761431.876732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761431.876735:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761431.876756:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761431.876766:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274162304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761431.876779:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162176:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761431.876834:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761431.876866:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761431/real 1787761431] req@ffff9ac451c52580 x1874603274162176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761431.876879:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274162176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761431.876886:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761431.876906:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761431.876910:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162176:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761431.876930:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162304:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761431.876947:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761431.876956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761431/real 1787761431] req@ffff9ac451c50f00 x1874603274162304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761431.876965:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274162304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761431.876970:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761431.876979:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761431.876981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162304:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761432.452680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761432.900599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761433.476673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761433.924615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761434.500662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761434.948596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761435.524685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761435.972610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761436.548649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761436.996602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761436.996622:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761436.996633:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761436.996637:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761436.996646:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 481 00000100:00080000:0.0:1787761436.996655:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761436.996661:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761436.996691:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274162432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761436.996714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761436.996717:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761436.996718:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761436.996721:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 481 00000100:00080000:0.0:1787761436.996723:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761436.996726:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761436.996733:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761436.996742:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274162560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761436.996768:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162432:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761436.996825:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761436.996877:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761436/real 1787761436] req@ffff9ac451f861c0 x1874603274162432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761436.996891:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274162432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761436.996899:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761436.996902:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761436.996905:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162432:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761436.996910:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162560:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761436.996926:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761436.996934:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761436/real 1787761436] req@ffff9ac451f843c0 x1874603274162560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761436.996942:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274162560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761436.996947:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761436.996949:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761436.996951:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162560:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761437.572648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761438.020590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761438.596721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761439.044623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761439.620817:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761440.068633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761440.644725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761441.092662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761441.668819:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761442.116642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761442.116647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761442.116676:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761442.116681:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761442.116693:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 486 00000100:00080000:0.0:1787761442.116706:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761442.116716:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761442.116757:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274162688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761442.116787:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761442.116791:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761442.116793:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761442.116798:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 486 00000100:00080000:0.0:1787761442.116802:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761442.116807:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761442.116834:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761442.116869:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761442.116875:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274162816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761442.116975:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761442.117027:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761442/real 1787761442] req@ffff9ac451f85680 x1874603274162688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761442.117051:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274162688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761442.117066:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761442.117071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761442.117077:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761442.117107:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162816:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761442.117134:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761442.117149:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761442/real 1787761442] req@ffff9ac451f843c0 x1874603274162816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761442.117165:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274162816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761442.117193:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761442.117197:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761442.117200:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274162816:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761442.692652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761443.140602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761443.716640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761444.164609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761444.740643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761445.188581:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761445.764638:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761446.212593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761446.788662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761447.236581:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761447.236611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761447.236617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761447.236620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761447.236628:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 491 00000100:00080000:2.0:1787761447.236637:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761447.236657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761447.236685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274162944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761447.236709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761447.236713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761447.236715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761447.236720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 491 00000100:00080000:2.0:1787761447.236725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761447.236730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761447.236741:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761447.236746:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162944:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761447.236758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274163072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761447.236823:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761447.236861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761447/real 1787761447] req@ffff9ac4523bc3c0 x1874603274162944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761447.236878:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274162944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761447.236895:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761447.236900:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761447.236904:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274162944:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761447.236926:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163072:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761447.236950:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761447.236963:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761447/real 1787761447] req@ffff9ac4523be940 x1874603274163072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761447.236976:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274163072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761447.236983:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761447.236987:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761447.236990:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163072:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761447.812669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761448.260588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761448.836714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761449.284597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761449.860645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761450.308616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761450.884673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761451.332613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761451.908688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761452.356617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761452.356669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761452.356674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761452.356678:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761452.356685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 497 00000100:00080000:3.0:1787761452.356694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761452.356700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761452.356733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274163200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761452.356773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761452.356776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761452.356778:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761452.356780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 497 00000100:00080000:3.0:1787761452.356783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761452.356785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761452.356794:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761452.356803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274163328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761452.356833:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163200:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761452.356896:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761452.356933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761452/real 1787761452] req@ffff9ac451c50f00 x1874603274163200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761452.356946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274163200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761452.356954:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761452.356971:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761452.356974:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163200:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761452.356980:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163328:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761452.356997:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761452.357006:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761452/real 1787761452] req@ffff9ac451c52580 x1874603274163328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761452.357014:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274163328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761452.357020:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761452.357022:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761452.357024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163328:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761452.932666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761453.380596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761453.956655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761454.404598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761454.980683:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761455.428614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761456.004674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761456.452621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761457.028671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761457.476610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761457.476640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761457.476649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761457.476654:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761457.476665:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 502 00000100:00080000:2.0:1787761457.476679:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761457.476687:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761457.476748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274163456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761457.476777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761457.476781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761457.476783:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761457.476788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 502 00000100:00080000:2.0:1787761457.476811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761457.476816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761457.476829:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761457.476831:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761457.476846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274163584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761457.476924:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761457.476974:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761457/real 1787761457] req@ffff9ac4523bda40 x1874603274163456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761457.476992:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274163456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761457.477004:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761457.477009:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761457.477013:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761457.477053:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163584:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761457.477080:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761457.477095:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761457/real 1787761457] req@ffff9ac4523bf840 x1874603274163584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761457.477108:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274163584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761457.477117:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761457.477121:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761457.477124:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163584:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761458.052660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761458.500601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761459.076707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761459.524617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761460.100684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761460.548613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761461.124710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761461.572609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761462.148640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761462.596607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761462.596628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761462.596637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761462.596643:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761462.596655:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 507 00000100:00080000:3.0:1787761462.596705:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761462.596713:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761462.596756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274163712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761462.596789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761462.596793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761462.596796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761462.596800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 507 00000100:00080000:3.0:1787761462.596804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761462.596809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761462.596821:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761462.596837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274163840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761462.596839:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163712:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761462.596917:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761462.596956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761462/real 1787761462] req@ffff9ac451c503c0 x1874603274163712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761462.596970:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274163712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761462.596979:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761462.596982:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761462.596985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163712:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761462.597007:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163840:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761462.597043:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761462.597053:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761462/real 1787761462] req@ffff9ac451c50780 x1874603274163840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761462.597062:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274163840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761462.597068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761462.597070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761462.597072:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274163840:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761463.172722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761463.620596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761464.196691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761464.644608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761465.220671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761465.668610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761466.244668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761466.692606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761467.268678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761467.716611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761467.716656:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761467.716662:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761467.716667:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761467.716677:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 512 00000100:00080000:3.0:1787761467.716688:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761467.716696:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761467.716737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274163968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761467.716761:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761467.716764:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761467.716766:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761467.716784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 512 00000100:00080000:3.0:1787761467.716788:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761467.716792:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761467.716802:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761467.716814:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787761467.716815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274164096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761467.716927:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164096:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761468.292688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761468.740720:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761469.316671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761469.764663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761470.340652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761470.788772:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787761470.788884:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787761470.788927:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787761470.788945:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787761470.788984:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787761470.789044:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761467/real 1787761470] req@ffff9ac451c50b40 x1874603274164096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761470.789072:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274164096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761470.789084:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761470.789090:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761470.789095:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164096:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787761470.789104:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761467/real 1787761470] req@ffff9ac451c52940 x1874603274163968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761470.789138:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274163968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761470.789147:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761470.789152:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761470.789156:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274163968:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787761471.364652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761471.812666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761472.388682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761472.836646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761472.836656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761472.836657:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761472.836663:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761472.836675:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 517 00000100:00080000:0.0:1787761472.836685:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761472.836693:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761472.836727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274164224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761472.836756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761472.836760:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761472.836762:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761472.836766:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 517 00000100:00080000:0.0:1787761472.836770:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761472.836775:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761472.836793:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761472.836800:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164224:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761472.836809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274164352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761472.836868:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761472.836916:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761472/real 1787761472] req@ffff9ac451f861c0 x1874603274164224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761472.836937:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274164224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761472.836947:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761472.836952:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761472.836955:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164224:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761472.836978:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164352:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761472.837004:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761472.837017:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761472/real 1787761472] req@ffff9ac451f84780 x1874603274164352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761472.837047:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274164352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761472.837057:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761472.837060:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761472.837064:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164352:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761473.412680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761473.860705:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761474.436669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761474.884604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761475.460651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761475.908596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761476.484668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761476.932607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761477.508722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761477.956664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761477.956716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761477.956726:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761477.956731:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761477.956743:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 522 00000100:00080000:3.0:1787761477.956757:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761477.956766:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761477.956812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274164480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761477.956851:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761477.956856:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761477.956859:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761477.956864:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 522 00000100:00080000:3.0:1787761477.956868:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761477.956872:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761477.956884:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761477.956897:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274164608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761477.956976:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164480:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761477.957062:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761477.957116:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761477/real 1787761477] req@ffff9ac451c52d00 x1874603274164480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761477.957140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274164480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761477.957154:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761477.957159:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761477.957165:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164480:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761477.957174:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164608:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761477.957199:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761477.957213:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761477/real 1787761477] req@ffff9ac451c530c0 x1874603274164608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761477.957229:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274164608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761477.957267:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761477.957274:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761477.957278:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164608:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761478.532787:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761478.980607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761479.556802:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761480.004675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761480.580916:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761481.028647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761481.604670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761482.052683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761482.628666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761483.076670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761483.076680:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761483.076711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761483.076718:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761483.076731:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 527 00000100:00080000:3.0:1787761483.076745:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761483.076755:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761483.076800:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274164736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761483.076838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761483.076843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761483.076846:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761483.076851:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 527 00000100:00080000:3.0:1787761483.076855:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761483.076859:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761483.076872:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761483.076888:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274164864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761483.076914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164736:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761483.077018:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761483.077080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761483/real 1787761483] req@ffff9ac451c530c0 x1874603274164736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761483.077115:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274164736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761483.077136:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761483.077144:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761483.077152:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164736:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761483.077196:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164864:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761483.077234:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761483.077252:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761483/real 1787761483] req@ffff9ac451c52d00 x1874603274164864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761483.077269:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274164864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761483.077280:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761483.077285:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761483.077289:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274164864:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761483.652752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761484.100674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761484.676756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761485.124635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761485.700744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761486.148651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761486.724759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761487.172658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761487.748752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761488.196625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761488.196720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761488.196734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761488.196742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761488.196757:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 532 00000100:00080000:2.0:1787761488.196774:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761488.196783:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761488.196833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274164992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761488.196875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761488.196880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761488.196883:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761488.196889:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 532 00000100:00080000:2.0:1787761488.196893:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761488.196899:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761488.196913:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761488.196942:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761488.196942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274165120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761488.197037:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761488.197087:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761488/real 1787761488] req@ffff9ac4523bf840 x1874603274164992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761488.197110:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274164992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761488.197124:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761488.197130:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761488.197135:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274164992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761488.197165:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165120:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761488.197189:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761488.197204:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761488/real 1787761488] req@ffff9ac4523bda40 x1874603274165120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761488.197219:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274165120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761488.197252:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761488.197257:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761488.197261:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165120:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761488.772660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761489.220585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761489.796740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761490.244627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761490.820753:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761491.268638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761491.844804:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761492.292673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761492.868746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761493.316634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761493.316637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761493.316653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761493.316661:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761493.316678:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 538 00000100:00080000:2.0:1787761493.316696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761493.316709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761493.316766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274165248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761493.316825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761493.316831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761493.316834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761493.316840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 538 00000100:00080000:2.0:1787761493.316848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761493.316854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761493.316869:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761493.316892:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274165248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761493.316893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274165376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761493.317003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761493.317053:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761493/real 1787761493] req@ffff9ac4523be940 x1874603274165248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761493.317071:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274165248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761493.317082:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761493.317086:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761493.317089:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274165248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761493.317117:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165376:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761493.317135:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761493.317144:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761493/real 1787761493] req@ffff9ac4523bc3c0 x1874603274165376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761493.317153:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274165376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761493.317159:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761493.317176:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761493.317178:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165376:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761493.892778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761494.340656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761494.916762:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761495.364648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761495.940698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761496.388649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761496.964722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761497.412677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761497.988640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761498.436630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761498.436651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761498.436662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761498.436667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761498.436680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 543 00000100:00080000:2.0:1787761498.436695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761498.436703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761498.436751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274165504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761498.436780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761498.436784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761498.436787:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761498.436792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 543 00000100:00080000:2.0:1787761498.436797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761498.436802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761498.436837:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761498.436842:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274165504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761498.436850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274165632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761498.436929:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761498.436975:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761498/real 1787761498] req@ffff9ac4523bc000 x1874603274165504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761498.436990:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274165504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761498.436999:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761498.437002:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761498.437005:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274165504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761498.437030:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165632:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761498.437047:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761498.437056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761498/real 1787761498] req@ffff9ac4523bd680 x1874603274165632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761498.437064:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274165632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761498.437070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761498.437073:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761498.437075:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165632:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761499.012701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761499.460676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761500.036738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761500.484635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761501.060741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761501.508626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761502.084744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761502.532643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761503.108770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761503.556662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761503.556666:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761503.556679:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761503.556685:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761503.556698:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 548 00000100:00080000:1.0:1787761503.556712:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761503.556723:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761503.556769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274165760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761503.556807:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761503.556812:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761503.556815:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761503.556820:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 548 00000100:00080000:1.0:1787761503.556825:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761503.556829:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761503.556842:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761503.556858:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274165888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761503.556905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274165760:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761503.557004:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761503.557056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761503/real 1787761503] req@ffff9ac447843840 x1874603274165760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761503.557080:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274165760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761503.557095:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761503.557100:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761503.557105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274165760:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761503.557115:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165888:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761503.557139:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761503.557153:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761503/real 1787761503] req@ffff9ac447841e00 x1874603274165888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761503.557168:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274165888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761503.557178:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761503.557182:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761503.557185:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274165888:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761504.132786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761504.580652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761505.156742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761505.604655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761506.180725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761506.628673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761507.204709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761507.652621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761508.228758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761508.676667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761508.676679:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761508.676684:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787761508.676691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761508.676699:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 553 00000100:00080000:3.0:1787761508.676715:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761508.676727:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761508.676776:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274166016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761508.676843:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761508.676848:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761508.676851:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761508.676857:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 553 00000100:00080000:3.0:1787761508.676862:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761508.676867:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761508.676879:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761508.676895:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274166144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761508.676978:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166016:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761508.677077:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761508.677131:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761508/real 1787761508] req@ffff9ac451c52940 x1874603274166016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761508.677154:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274166016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761508.677168:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761508.677173:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761508.677178:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761508.677188:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166144:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761508.677213:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761508.677226:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761508/real 1787761508] req@ffff9ac451c50b40 x1874603274166144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761508.677241:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274166144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761508.677251:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761508.677255:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761508.677259:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166144:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761509.252869:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761509.700648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761510.276764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761510.724611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761511.300764:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761511.748660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761512.324756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761512.772698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761513.348779:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761513.796636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761513.796662:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761513.796672:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761513.796679:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761513.796691:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 558 00000100:00080000:3.0:1787761513.796706:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761513.796717:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761513.796761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274166272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761513.796802:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761513.796810:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761513.796813:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761513.796822:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 558 00000100:00080000:3.0:1787761513.796829:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761513.796838:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761513.796872:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761513.796891:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787761513.796896:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274166400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761513.797019:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761513.797071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761513/real 1787761513] req@ffff9ac451c50780 x1874603274166272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761513.797094:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274166272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761513.797107:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761513.797112:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761513.797118:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761513.797145:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166400:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761513.797174:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761513.797188:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761513/real 1787761513] req@ffff9ac451c503c0 x1874603274166400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761513.797203:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274166400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761513.797214:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761513.797218:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761513.797222:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166400:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761514.372716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761514.820655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761515.396720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761515.844628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761516.420714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761516.868640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761517.444770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761517.892692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761518.468756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761518.916694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761518.916707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761518.916713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787761518.916721:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761518.916727:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 563 00000100:00080000:1.0:1787761518.916746:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761518.916760:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761518.916819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274166528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761518.916865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761518.916870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761518.916874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761518.916884:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 563 00000100:00080000:1.0:1787761518.916893:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761518.916903:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761518.916920:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761518.916923:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761518.916956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274166656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761518.917013:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761518.917085:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761518/real 1787761518] req@ffff9ac447841a40 x1874603274166528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761518.917110:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274166528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761518.917124:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761518.917130:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761518.917135:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761518.917164:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166656:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761518.917193:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761518.917208:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761518/real 1787761518] req@ffff9ac4478412c0 x1874603274166656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761518.917223:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274166656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761518.917234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761518.917238:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761518.917242:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166656:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761519.492744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761519.940649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761520.516716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761520.964652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761521.540757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761521.988610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761522.564794:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761523.012686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761523.588726:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761524.036611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761524.036629:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761524.036637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761524.036643:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761524.036654:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 568 00000100:00080000:0.0:1787761524.036663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761524.036686:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761524.036727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274166784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761524.036749:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761524.036752:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761524.036756:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761524.036759:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 568 00000100:00080000:0.0:1787761524.036762:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761524.036765:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761524.036774:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761524.036783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274166912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761524.036791:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166784:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761524.036850:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761524.036893:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761524/real 1787761524] req@ffff9ac451f84000 x1874603274166784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761524.036912:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274166784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761524.036919:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761524.036923:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761524.036926:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274166784:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761524.036943:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166912:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761524.036961:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761524.036970:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761524/real 1787761524] req@ffff9ac451f870c0 x1874603274166912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761524.036991:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274166912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761524.036998:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761524.037000:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761524.037002:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274166912:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761524.612652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761525.060643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761525.636643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761526.084594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761526.660720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761527.108593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761527.684627:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761528.132590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761528.708660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761529.156597:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761529.156602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761529.156608:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761529.156613:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761529.156624:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 573 00000100:00080000:0.0:1787761529.156635:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761529.156642:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761529.156676:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274167040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761529.156698:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761529.156716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761529.156718:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761529.156723:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 573 00000100:00080000:0.0:1787761529.156728:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787761529.156730:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167040:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787761529.156732:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761529.156742:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761529.156755:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274167168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761529.156796:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761529.156834:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761529/real 1787761529] req@ffff9ac451f86940 x1874603274167040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761529.156847:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274167040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761529.156855:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761529.156858:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761529.156861:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167040:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761529.156878:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167168:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761529.156894:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761529.156902:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761529/real 1787761529] req@ffff9ac451f87c00 x1874603274167168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761529.156910:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274167168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761529.156926:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761529.156929:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761529.156931:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167168:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761529.732643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761530.180587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761530.756703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761531.204602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761531.780744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761532.228672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761532.804684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761533.252617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761533.828733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761534.276587:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761534.276595:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761534.276599:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787761534.276608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761534.276615:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 579 00000100:00080000:0.0:1787761534.276624:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761534.276631:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761534.276664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274167296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761534.276686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761534.276689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761534.276690:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761534.276693:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 579 00000100:00080000:0.0:1787761534.276695:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761534.276698:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761534.276718:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761534.276730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274167424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761534.276780:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761534.276959:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167424:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761534.852732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761535.300604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761535.876674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761536.324602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761536.900640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761537.348651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761537.348844:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787761537.348890:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761537.348922:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761537.348975:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761537.349052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761534/real 1787761537] req@ffff9ac451f84b40 x1874603274167296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761537.349082:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274167296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761537.349095:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761537.349100:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761537.349105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761537.349113:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761534/real 1787761537] req@ffff9ac451f87c00 x1874603274167424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761537.349125:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274167424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761537.349132:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761537.349136:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761537.349141:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167424:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761537.924642:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761538.372606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761538.948735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761539.396622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761539.396690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761539.396728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761539.396734:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761539.396749:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 584 00000100:00080000:2.0:1787761539.396765:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761539.396775:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761539.396828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274167552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761539.396878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761539.396886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761539.396891:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761539.396901:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 584 00000100:00080000:2.0:1787761539.396910:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761539.396920:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761539.396938:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167552:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787761539.396939:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761539.396981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274167680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761539.397028:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761539.397073:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761539/real 1787761539] req@ffff9ac4523bc780 x1874603274167552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761539.397088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274167552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761539.397097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761539.397101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761539.397104:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167552:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761539.397124:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167680:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761539.397161:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761539.397172:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761539/real 1787761539] req@ffff9ac4523be580 x1874603274167680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761539.397198:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274167680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761539.397205:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761539.397208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761539.397230:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167680:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761539.972848:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761540.420639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761540.996765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761541.444607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761542.020747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761542.468629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761543.044769:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761543.492610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761544.068700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761544.516599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761544.516627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761544.516633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761544.516636:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761544.516644:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 589 00000100:00080000:1.0:1787761544.516654:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761544.516660:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761544.516691:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274167808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761544.516713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761544.516716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761544.516718:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761544.516720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 589 00000100:00080000:1.0:1787761544.516723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761544.516725:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761544.516746:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761544.516759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274167936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761544.516764:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167808:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761544.516821:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761544.516856:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761544/real 1787761544] req@ffff9ac447843480 x1874603274167808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761544.516869:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274167808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761544.516877:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761544.516880:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761544.516883:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274167808:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761544.516899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167936:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761544.516914:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761544.516922:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761544/real 1787761544] req@ffff9ac4478412c0 x1874603274167936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761544.516931:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274167936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761544.516936:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761544.516939:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761544.516941:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274167936:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761545.092671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761545.540641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761546.116828:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761546.564678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761547.140653:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761547.588613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761548.164678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761548.612585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761549.188744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761549.636663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761549.636664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761549.636679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761549.636685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761549.636698:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 594 00000100:00080000:1.0:1787761549.636713:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761549.636723:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761549.636771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274168064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761549.636807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761549.636813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761549.636815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761549.636821:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 594 00000100:00080000:1.0:1787761549.636825:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761549.636830:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761549.636854:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761549.636878:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761549.636879:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274168192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761549.636987:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761549.637037:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761549/real 1787761549] req@ffff9ac447841a40 x1874603274168064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761549.637060:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274168064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761549.637104:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761549.637110:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761549.637127:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761549.637157:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168192:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761549.637189:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761549.637206:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761549/real 1787761549] req@ffff9ac447841e00 x1874603274168192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761549.637225:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274168192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761549.637242:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761549.637248:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761549.637254:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168192:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761550.212756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761550.660648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761551.236806:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761551.684629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761552.260790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761552.708629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761553.284777:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761553.732680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761554.308829:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761554.756661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761554.756662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761554.756678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761554.756685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761554.756699:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 599 00000100:00080000:1.0:1787761554.756730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761554.756741:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761554.756801:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274168320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761554.756837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761554.756841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761554.756844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761554.756848:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 599 00000100:00080000:1.0:1787761554.756853:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761554.756858:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761554.756870:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761554.756888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274168448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761554.756921:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168320:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761554.757018:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761554.757101:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761554/real 1787761554] req@ffff9ac447843840 x1874603274168320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761554.757126:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274168320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761554.757140:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761554.757146:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761554.757151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761554.757180:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168448:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761554.757211:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761554.757226:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761554/real 1787761554] req@ffff9ac447842580 x1874603274168448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761554.757241:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274168448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761554.757252:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761554.757256:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761554.757260:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168448:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761555.332715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761555.780654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761556.356736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761556.804596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761557.380711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761557.828635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761558.404733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761558.852664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761559.428714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761559.876671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761559.876727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761559.876737:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761559.876761:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761559.876772:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 604 00000100:00080000:0.0:1787761559.876787:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761559.876796:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761559.876842:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274168576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761559.876881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761559.876886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761559.876889:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761559.876894:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 604 00000100:00080000:0.0:1787761559.876899:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761559.876906:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761559.876922:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761559.876946:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274168704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761559.877018:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168576:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761559.877121:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761559.877185:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761559/real 1787761559] req@ffff9ac451f86940 x1874603274168576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761559.877231:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274168576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761559.877272:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761559.877295:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761559.877304:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761559.877318:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168704:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761559.877369:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761559.877388:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761559/real 1787761559] req@ffff9ac451f870c0 x1874603274168704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761559.877404:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274168704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761559.877415:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761559.877419:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761559.877424:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168704:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761560.452818:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761560.900666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761561.476800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761561.924686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761562.500745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761562.948659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761563.524763:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761563.972681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761564.548758:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761564.996633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761564.996731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761564.996743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761564.996749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761564.996762:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 609 00000100:00080000:1.0:1787761564.996776:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761564.996785:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761564.996831:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274168832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761564.996879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761564.996887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761564.996893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787761564.996918:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168832:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787761564.996931:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 609 00000100:00080000:1.0:1787761564.996942:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761564.996955:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761564.996995:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761564.997029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274168960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761564.997038:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761564.997111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761564/real 1787761564] req@ffff9ac447840000 x1874603274168832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761564.997129:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274168832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761564.997139:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761564.997143:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761564.997146:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274168832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761564.997173:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168960:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761564.997209:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761564.997219:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761564/real 1787761564] req@ffff9ac447840f00 x1874603274168960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761564.997228:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274168960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761564.997234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761564.997237:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761564.997239:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274168960:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761565.572774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761566.020608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761566.596679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761567.044598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761567.620645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761568.068617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761568.644744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761569.092608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761569.668717:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761570.116619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761570.116630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761570.116636:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761570.116647:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 614 00000100:00080000:1.0:1787761570.116662:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761570.116671:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761570.116693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787761570.116730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274169088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761570.116761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761570.116765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761570.116766:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761570.116770:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 614 00000100:00080000:1.0:1787761570.116773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761570.116775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761570.116786:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761570.116795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274169216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761570.116841:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169088:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761570.116921:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761570.116986:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761570/real 1787761570] req@ffff9ac447840780 x1874603274169088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761570.117002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274169088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761570.117012:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761570.117015:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761570.117018:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761570.117024:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169216:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761570.117043:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761570.117051:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761570/real 1787761570] req@ffff9ac447841680 x1874603274169216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761570.117082:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274169216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761570.117088:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761570.117091:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761570.117093:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169216:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761570.692700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761571.140610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761571.716750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761572.164669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761572.740762:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761573.188652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761573.764735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761574.212635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761574.788707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761575.236642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761575.236705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761575.236767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761575.236774:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761575.236790:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 619 00000100:00080000:1.0:1787761575.236806:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761575.236832:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761575.236884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274169344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761575.236945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761575.236950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761575.236953:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761575.236959:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 619 00000100:00080000:1.0:1787761575.236965:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761575.236970:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761575.236986:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761575.237013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274169472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761575.237032:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169344:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761575.237142:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761575.237194:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761575/real 1787761575] req@ffff9ac4478403c0 x1874603274169344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761575.237217:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274169344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761575.237231:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761575.237236:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761575.237242:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761575.237269:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169472:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761575.237295:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761575.237309:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761575/real 1787761575] req@ffff9ac4478421c0 x1874603274169472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761575.237324:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274169472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761575.237355:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761575.237360:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761575.237364:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761575.812787:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761576.260652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761576.836727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761577.284642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761577.860768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761578.308656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761578.884721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761579.332674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761579.908788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761580.356630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761580.356685:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761580.356721:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761580.356726:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761580.356739:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 625 00000100:00080000:0.0:1787761580.356752:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761580.356761:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761580.356807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274169600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761580.356847:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761580.356872:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761580.356875:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761580.356893:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 625 00000100:00080000:0.0:1787761580.356897:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761580.356903:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761580.356916:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761580.356935:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274169728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761580.356942:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169600:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761580.357062:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761580.357113:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761580/real 1787761580] req@ffff9ac451f84000 x1874603274169600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761580.357131:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274169600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761580.357142:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761580.357147:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761580.357151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761580.357197:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169728:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761580.357215:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761580.357224:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761580/real 1787761580] req@ffff9ac451f84780 x1874603274169728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761580.357232:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274169728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761580.357238:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761580.357241:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761580.357243:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169728:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761580.932728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761581.380636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761581.956672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761582.404622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761582.980690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761583.428600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761584.004696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761584.452637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761585.028746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761585.476609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761585.476640:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761585.476649:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761585.476655:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761585.476668:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 630 00000100:00080000:1.0:1787761585.476685:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761585.476695:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761585.476745:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274169856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761585.476805:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761585.476814:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761585.476831:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761585.476838:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 630 00000100:00080000:1.0:1787761585.476844:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761585.476851:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761585.476865:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761585.476868:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761585.476883:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274169984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761585.476947:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761585.476989:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761585/real 1787761585] req@ffff9ac4478430c0 x1874603274169856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761585.477004:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274169856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761585.477013:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761585.477017:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761585.477020:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274169856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761585.477044:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169984:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761585.477060:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761585.477068:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761585/real 1787761585] req@ffff9ac447842d00 x1874603274169984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761585.477076:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274169984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761585.477083:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761585.477085:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761585.477088:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274169984:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761586.052649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761586.500595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761587.076671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761587.524599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761588.100733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761588.548661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761589.124720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761589.572670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761590.148662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761590.596622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761590.596683:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761590.596696:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761590.596702:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761590.596715:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 635 00000100:00080000:1.0:1787761590.596730:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761590.596740:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761590.596816:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274170112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761590.596853:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761590.596858:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761590.596862:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761590.596869:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 635 00000100:00080000:1.0:1787761590.596875:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761590.596883:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761590.596897:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170112:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787761590.596920:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761590.596945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274170240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761590.596998:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761590.597048:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761590/real 1787761590] req@ffff9ac447842940 x1874603274170112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761590.597079:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274170112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761590.597089:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761590.597093:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761590.597096:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170112:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761590.597124:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274170240:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761590.597144:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761590.597154:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761590/real 1787761590] req@ffff9ac447843c00 x1874603274170240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761590.597163:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274170240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761590.597169:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761590.597171:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761590.597181:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274170240:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761591.172728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761591.620657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761592.196813:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761592.644648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761593.220714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761593.668625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761594.244743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761594.692673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761595.268829:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761595.716688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761595.716718:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761595.716729:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761595.716779:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761595.716795:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 640 00000100:00080000:0.0:1787761595.716811:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761595.716824:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761595.716877:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274170368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761595.716917:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761595.716922:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761595.716925:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761595.716930:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 640 00000100:00080000:0.0:1787761595.716935:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761595.716940:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761595.716953:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761595.716972:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274170496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761595.716990:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170368:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761595.717079:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761595.717144:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761595/real 1787761595] req@ffff9ac451f861c0 x1874603274170368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761595.717172:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274170368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761595.717207:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761595.717212:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761595.717218:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761595.717249:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274170496:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761595.717283:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761595.717299:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761595/real 1787761595] req@ffff9ac451f843c0 x1874603274170496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761595.717314:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274170496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761595.717324:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761595.717328:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761595.717332:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274170496:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761596.292786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761596.740641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761597.316684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761597.764638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761598.340690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761598.788637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761599.364791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761599.812645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761600.388703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761600.836609:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761600.836621:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1787761600.836623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761600.836626:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761600.836637:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 645 00000100:00080000:0.0:1787761600.836651:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761600.836660:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761600.836705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274170624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761600.836757:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761600.836760:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761600.836762:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761600.836766:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 645 00000100:00080000:0.0:1787761600.836769:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761600.836773:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761600.836783:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761600.836797:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274170752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761600.836813:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761600.836936:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274170752:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761601.412794:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761601.860627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761602.436672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761602.884585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761603.460710:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761603.908593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761603.908843:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787761603.908890:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761603.908910:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761603.908949:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787761603.909019:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761600/real 1787761603] req@ffff9ac451f843c0 x1874603274170752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761603.909051:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274170752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761603.909067:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761603.909074:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761603.909102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274170752:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787761603.909112:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761600/real 1787761603] req@ffff9ac451f85680 x1874603274170624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761603.909128:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274170624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761603.909136:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761603.909141:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761603.909144:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170624:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787761604.484663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761604.932616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761605.508785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761605.956668:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761605.956697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761605.956704:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761605.956714:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761605.956728:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 650 00000100:00080000:0.0:1787761605.956741:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761605.956752:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761605.956797:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274170880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761605.956825:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761605.956831:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761605.956834:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761605.956840:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 650 00000100:00100000:2.0:1787761605.956850:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170880:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787761605.956866:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761605.956876:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761605.956891:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761605.956914:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274171008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761605.956952:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761605.957010:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761605/real 1787761605] req@ffff9ac451f861c0 x1874603274170880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761605.957034:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274170880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761605.957048:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761605.957053:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761605.957059:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274170880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761605.957087:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171008:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761605.957115:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761605.957130:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761605/real 1787761605] req@ffff9ac451f84780 x1874603274171008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761605.957197:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274171008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761605.957214:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761605.957221:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761605.957228:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171008:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761606.532792:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761606.980664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761607.556722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761608.004654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761608.580667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761609.028698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761609.604771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761610.052626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761610.629633:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761611.076600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761611.076608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761611.076612:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761611.076621:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 655 00000100:00080000:3.0:1787761611.076630:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761611.076637:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761611.076662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787761611.076682:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274171136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761611.076705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761611.076708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761611.076709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761611.076713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 655 00000100:00080000:3.0:1787761611.076716:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761611.076720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761611.076729:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761611.076733:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787761611.076742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274171264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761611.076792:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761611.076825:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761611/real 1787761611] req@ffff9ac451c52580 x1874603274171136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761611.076838:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274171136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761611.076846:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761611.076848:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761611.076851:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761611.076868:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171264:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761611.076883:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761611.076890:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761611/real 1787761611] req@ffff9ac451c50f00 x1874603274171264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761611.076898:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274171264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761611.076914:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761611.076917:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761611.076919:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171264:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761611.653660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761612.100707:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761612.676770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761613.124728:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761613.700830:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761614.148748:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761614.724779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761615.172742:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761615.748760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761616.196684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761616.196708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761616.196718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761616.196724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761616.196737:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 660 00000100:00080000:1.0:1787761616.196751:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761616.196760:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761616.196807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274171392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761616.196845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761616.196850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761616.196853:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761616.196858:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 660 00000100:00080000:1.0:1787761616.196863:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761616.196869:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761616.196882:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761616.196907:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761616.196926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274171520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761616.197000:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761616.197052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761616/real 1787761616] req@ffff9ac447840b40 x1874603274171392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761616.197075:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274171392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761616.197090:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761616.197095:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761616.197101:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761616.197129:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171520:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761616.197172:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761616.197189:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761616/real 1787761616] req@ffff9ac447843c00 x1874603274171520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761616.197205:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274171520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761616.197237:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761616.197242:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761616.197246:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171520:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761616.772795:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761617.220641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761617.796771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761618.244661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761618.820753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761619.268655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761619.844781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761620.292653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761620.868757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761621.316642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761621.316726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761621.316736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761621.316742:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761621.316754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 666 00000100:00080000:0.0:1787761621.316767:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761621.316776:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761621.316822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274171648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761621.316859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761621.316864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761621.316867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761621.316872:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 666 00000100:00080000:0.0:1787761621.316876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761621.316880:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761621.316894:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761621.316911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274171776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761621.316928:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171648:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761621.317017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761621.317083:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761621/real 1787761621] req@ffff9ac451f84000 x1874603274171648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761621.317111:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274171648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761621.317128:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761621.317135:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761621.317141:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761621.317169:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171776:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761621.317198:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761621.317212:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761621/real 1787761621] req@ffff9ac451f870c0 x1874603274171776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761621.317228:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274171776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761621.317239:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761621.317243:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761621.317247:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274171776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761621.892756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761622.340631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761622.916772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761623.364645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761623.940731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761624.388658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761624.964779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761625.412635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761625.988742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761626.436635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761626.436635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761626.436674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761626.436679:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761626.436691:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 671 00000100:00080000:0.0:1787761626.436705:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761626.436715:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761626.436760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274171904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761626.436790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761626.436793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761626.436795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761626.436799:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 671 00000100:00080000:0.0:1787761626.436801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761626.436804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761626.436815:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761626.436829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274172032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761626.436870:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171904:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761626.436979:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761626.437055:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761626/real 1787761626] req@ffff9ac451f86940 x1874603274171904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761626.437083:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274171904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761626.437098:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761626.437104:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761626.437109:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274171904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761626.437120:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172032:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761626.437154:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761626.437170:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761626/real 1787761626] req@ffff9ac451f87c00 x1874603274172032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761626.437185:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274172032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761626.437195:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761626.437199:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761626.437204:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761627.012753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761627.460654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761628.036723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761628.484641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761629.060756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761629.508652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761630.084795:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761630.532637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761631.108679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787761631.556625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761631.556633:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761631.556637:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761631.556663:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 676 00000100:00080000:2.0:1787761631.556674:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761631.556681:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761631.556684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787761631.556715:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274172160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761631.556744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761631.556749:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761631.556751:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761631.556756:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 676 00000100:00080000:2.0:1787761631.556759:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761631.556763:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761631.556773:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761631.556787:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761631.556787:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274172288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761631.556850:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761631.556888:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761631/real 1787761631] req@ffff9ac4523bcf00 x1874603274172160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761631.556901:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274172160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761631.556909:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761631.556917:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761631.556920:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761631.556940:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172288:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761631.556957:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761631.556966:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761631/real 1787761631] req@ffff9ac4523be1c0 x1874603274172288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761631.556974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274172288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761631.556979:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761631.556982:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761631.556984:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172288:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761632.132680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761632.580599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761633.156663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761633.604591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761634.180657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761634.628622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761635.204664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761635.652590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761636.228748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787761636.676593:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761636.676595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761636.676603:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761636.676606:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761636.676614:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 681 00000100:00080000:2.0:1787761636.676623:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761636.676629:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761636.676660:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274172416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761636.676683:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761636.676687:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761636.676704:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761636.676709:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 681 00000100:00080000:2.0:1787761636.676714:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761636.676718:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761636.676729:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761636.676745:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274172544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761636.676765:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172416:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761636.676818:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761636.676853:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761636/real 1787761636] req@ffff9ac4523bf0c0 x1874603274172416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761636.676866:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274172416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761636.676873:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761636.676876:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761636.676879:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761636.676884:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172544:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761636.676909:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761636.676927:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761636/real 1787761636] req@ffff9ac4523be1c0 x1874603274172544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761636.676936:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274172544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761636.676941:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761636.676944:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761636.676946:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761637.252657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761637.700645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761638.276773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761638.724659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761639.300660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761639.748606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761640.324647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761640.772591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761641.348657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761641.796658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761641.796690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761641.796700:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761641.796706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761641.796720:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 686 00000100:00080000:0.0:1787761641.796734:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761641.796743:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761641.796790:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274172672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761641.796832:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761641.796837:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761641.796840:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761641.796845:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 686 00000100:00080000:0.0:1787761641.796849:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761641.796855:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761641.796868:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761641.796899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761641.796913:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274172800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761641.796992:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761641.797057:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761641/real 1787761641] req@ffff9ac451f84b40 x1874603274172672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761641.797084:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274172672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761641.797099:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761641.797104:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761641.797109:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761641.797136:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172800:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761641.797165:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761641.797181:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761641/real 1787761641] req@ffff9ac451f87c00 x1874603274172800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761641.797197:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274172800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761641.797228:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761641.797233:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761641.797237:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274172800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761642.372678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761642.820615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761643.396668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761643.844616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761644.420713:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761644.868649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761645.444783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761645.892676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761646.468751:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761646.916652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761646.916688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761646.916696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761646.916701:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761646.916712:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 691 00000100:00080000:1.0:1787761646.916722:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761646.916729:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761646.916766:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274172928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761646.916794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761646.916798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761646.916800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761646.916817:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 691 00000100:00080000:1.0:1787761646.916821:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761646.916825:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761646.916836:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761646.916850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274173056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761646.916861:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172928:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761646.916946:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761646.917008:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761646/real 1787761646] req@ffff9ac447842940 x1874603274172928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761646.917038:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274172928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761646.917053:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761646.917060:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761646.917066:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274172928:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761646.917096:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173056:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761646.917132:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761646.917151:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761646/real 1787761646] req@ffff9ac447842d00 x1874603274173056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761646.917171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274173056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761646.917184:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761646.917189:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761646.917194:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173056:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761647.492696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761647.940646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761648.516664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761648.964665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761649.540856:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761649.988692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761650.564746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761651.012634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761651.588791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761652.036654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761652.036655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761652.036673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761652.036681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761652.036698:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 696 00000100:00080000:1.0:1787761652.036711:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761652.036725:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761652.036781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274173184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761652.036838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761652.036844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761652.036848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761652.036856:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 696 00000100:00080000:1.0:1787761652.036861:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761652.036868:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761652.036882:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761652.036893:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761652.036906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274173312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761652.036993:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761652.037051:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761652/real 1787761652] req@ffff9ac4478430c0 x1874603274173184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761652.037069:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274173184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761652.037081:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761652.037085:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761652.037089:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761652.037120:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173312:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761652.037138:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761652.037148:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761652/real 1787761652] req@ffff9ac4478421c0 x1874603274173312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761652.037156:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274173312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761652.037162:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761652.037165:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761652.037167:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761652.612702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761653.060654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761653.636687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761654.084629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761654.660696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761655.108646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761655.684712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761656.132664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761656.708742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761657.156653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761657.156747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761657.156757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761657.156762:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761657.156774:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 701 00000100:00080000:0.0:1787761657.156787:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761657.156796:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761657.156844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274173440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761657.156884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761657.156889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761657.156892:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761657.156897:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 701 00000100:00080000:0.0:1787761657.156901:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761657.156905:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761657.156931:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761657.156951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274173568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761657.156980:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173440:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761657.157074:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761657.157133:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761657/real 1787761657] req@ffff9ac451f86940 x1874603274173440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761657.157171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274173440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761657.157183:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761657.157187:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761657.157190:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761657.157219:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173568:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761657.157238:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761657.157248:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761657/real 1787761657] req@ffff9ac451f870c0 x1874603274173568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761657.157256:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274173568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761657.157262:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761657.157265:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761657.157267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173568:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761657.732772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761658.180638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761658.756742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761659.204647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761659.780779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761660.228630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761660.804745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761661.252671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761661.828742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761662.276590:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761662.276598:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761662.276602:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761662.276614:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 707 00000100:00100000:3.0:1787761662.276615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761662.276625:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761662.276651:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761662.276682:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274173696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761662.276704:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761662.276707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761662.276709:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761662.276712:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 707 00000100:00080000:0.0:1787761662.276715:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761662.276717:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761662.276725:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761662.276734:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274173824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761662.276742:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173696:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761662.276799:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761662.276830:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761662/real 1787761662] req@ffff9ac451f84000 x1874603274173696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761662.276843:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274173696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761662.276861:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761662.276865:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761662.276867:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173696:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761662.276886:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173824:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761662.276903:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761662.276914:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761662/real 1787761662] req@ffff9ac451f84780 x1874603274173824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761662.276927:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274173824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761662.276935:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761662.276939:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761662.276942:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274173824:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761662.852696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761663.300588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761663.876664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761664.324603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761664.900657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761665.348641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761665.924680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761666.372647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761666.948721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761667.396664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761667.396727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761667.396738:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761667.396743:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761667.396755:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 712 00000100:00080000:2.0:1787761667.396769:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761667.396778:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761667.396827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274173952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761667.396873:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761667.396897:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761667.396900:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761667.396907:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 712 00000100:00080000:2.0:1787761667.396914:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761667.396921:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761667.396935:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761667.396949:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761667.396960:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274174080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761667.397092:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174080:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761667.972708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761668.420621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761668.996745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761669.444647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761670.020751:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761670.468640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787761670.468897:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787761670.468948:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787761670.468971:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787761670.469034:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787761670.469134:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761667/real 1787761670] req@ffff9ac4523bcb40 x1874603274174080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761670.469181:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274174080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761670.469201:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761670.469210:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761670.469218:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174080:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787761670.469260:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761667/real 1787761670] req@ffff9ac4523bcf00 x1874603274173952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761670.469311:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274173952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761670.469328:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761670.469336:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761670.469357:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274173952:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787761671.044771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761671.492614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761672.068787:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761672.516655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761672.516728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761672.516739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761672.516744:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761672.516756:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 717 00000100:00080000:0.0:1787761672.516770:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761672.516779:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761672.516824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274174208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761672.516871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761672.516878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761672.516882:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761672.516891:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 717 00000100:00080000:0.0:1787761672.516900:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761672.516909:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761672.516926:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761672.516928:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761672.516962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274174336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787761672.517023:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761672.517100:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761672/real 1787761672] req@ffff9ac451f861c0 x1874603274174208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761672.517133:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274174208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761672.517164:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761672.517173:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761672.517181:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761672.517249:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174336:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761672.517359:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761672.517393:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761672/real 1787761672] req@ffff9ac451f85680 x1874603274174336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761672.517419:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274174336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761672.517435:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761672.517441:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761672.517447:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174336:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761673.092756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761673.540627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761674.116746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761674.564633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761675.141026:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761675.588651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761676.164741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761676.612658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761677.188740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761677.636651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761677.636744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761677.636753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761677.636758:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761677.636771:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 722 00000100:00080000:1.0:1787761677.636784:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761677.636793:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761677.636841:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274174464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761677.636880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761677.636886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761677.636888:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761677.636893:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 722 00000100:00080000:1.0:1787761677.636898:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761677.636919:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761677.636933:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761677.636952:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274174592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761677.636970:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174464:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761677.637065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761677.637117:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761677/real 1787761677] req@ffff9ac4478403c0 x1874603274174464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761677.637140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274174464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761677.637154:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761677.637160:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761677.637165:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761677.637223:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174592:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761677.637257:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761677.637273:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761677/real 1787761677] req@ffff9ac447841680 x1874603274174592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761677.637312:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274174592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761677.637328:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761677.637334:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761677.637341:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174592:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761678.212768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761678.660627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761679.236796:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761679.684658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761680.260878:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761680.708599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761681.284673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761681.732603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761682.308740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761682.756600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761682.756620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761682.756626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761682.756631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761682.756648:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 727 00000100:00080000:0.0:1787761682.756661:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761682.756669:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761682.756713:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274174720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761682.756749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761682.756754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761682.756756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761682.756761:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 727 00000100:00080000:0.0:1787761682.756766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761682.756771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761682.756782:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761682.756796:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274174848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761682.756817:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174720:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761682.756906:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761682.756955:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761682/real 1787761682] req@ffff9ac451f843c0 x1874603274174720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761682.756971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274174720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761682.756980:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761682.756983:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761682.756987:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761682.757009:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174848:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761682.757025:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761682.757034:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761682/real 1787761682] req@ffff9ac451f85680 x1874603274174848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761682.757042:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274174848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761682.757068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761682.757071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761682.757074:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274174848:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761683.332670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761683.780605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761684.356741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761684.804642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761685.380761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761685.828641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761686.404733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761686.852699:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761687.428747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761687.876639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761687.876699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761687.876710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761687.876716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761687.876730:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 732 00000100:00080000:1.0:1787761687.876748:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761687.876762:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761687.876818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274174976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761687.876861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761687.876866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761687.876869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761687.876877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 732 00000100:00080000:1.0:1787761687.876886:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761687.876894:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761687.876910:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761687.876937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274175104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761687.876981:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174976:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761687.877085:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761687.877190:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761687/real 1787761687] req@ffff9ac447840780 x1874603274174976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761687.877218:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274174976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761687.877234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761687.877239:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761687.877244:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274174976:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761687.877274:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175104:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761687.877306:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761687.877321:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761687/real 1787761687] req@ffff9ac447840f00 x1874603274175104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761687.877336:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274175104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761687.877347:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761687.877351:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761687.877355:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175104:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761688.452792:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761688.900632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761689.476805:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761689.924633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761690.500732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761690.948584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761691.524666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761691.972617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761692.548667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761692.996610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761692.996610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761692.996621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761692.996627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761692.996637:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 737 00000100:00080000:1.0:1787761692.996650:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761692.996658:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761692.996702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274175232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761692.996727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761692.996730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761692.996731:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761692.996735:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 737 00000100:00080000:1.0:1787761692.996737:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761692.996740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761692.996750:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761692.996758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274175360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787761692.996864:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274175232:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787761692.996962:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761692.997029:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761692/real 1787761692] req@ffff9ac447840000 x1874603274175232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761692.997064:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274175232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761692.997116:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761692.997126:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761692.997135:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274175232:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761692.997151:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175360:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787761692.997188:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787761692.997206:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761692/real 1787761692] req@ffff9ac447842580 x1874603274175360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787761692.997223:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274175360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787761692.997234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787761692.997238:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787761692.997243:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175360:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761693.572731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761694.020634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761694.596682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761695.044641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761695.620808:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761696.068646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761696.644756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761697.092721:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761697.668752:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761698.116631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761698.116700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761698.116711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761698.116716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761698.116749:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 742 00000100:00080000:2.0:1787761698.116762:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761698.116782:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761698.116827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274175488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761698.116879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761698.116884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761698.116887:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761698.116893:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 742 00000100:00080000:2.0:1787761698.116897:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761698.116901:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761698.116913:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761698.116930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274175616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761698.116953:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274175488:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761698.117076:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761698.117130:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761698/real 1787761698] req@ffff9ac4523be580 x1874603274175488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761698.117154:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274175488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761698.117180:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761698.117185:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761698.117191:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274175488:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761698.117222:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175616:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761698.117256:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761698.117280:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761698/real 1787761698] req@ffff9ac4523bc780 x1874603274175616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761698.117296:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274175616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761698.117307:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761698.117311:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761698.117315:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175616:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761698.692757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761699.140613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761699.716785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761700.164618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761700.740810:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761701.188662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761701.764740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761702.212628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761702.788759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761703.236642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761703.236662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761703.236673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761703.236678:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761703.236692:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 747 00000100:00080000:2.0:1787761703.236705:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761703.236716:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761703.236763:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274175744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761703.236799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761703.236804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761703.236822:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761703.236828:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 747 00000100:00080000:2.0:1787761703.236835:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761703.236843:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761703.236857:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761703.236885:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274175744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761703.236886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274175872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761703.236984:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761703.237073:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761703/real 1787761703] req@ffff9ac4523bd680 x1874603274175744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761703.237109:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274175744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761703.237130:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761703.237137:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761703.237146:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274175744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761703.237208:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175872:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761703.237253:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761703.237273:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761703/real 1787761703] req@ffff9ac4523bc000 x1874603274175872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761703.237290:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274175872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761703.237301:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761703.237306:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761703.237310:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274175872:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761703.812736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761704.260646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761704.836722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761705.284645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761705.860760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761706.308659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761706.884745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761707.332672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761707.908742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761708.356641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761708.356694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761708.356704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761708.356709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761708.356721:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 753 00000100:00080000:2.0:1787761708.356735:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761708.356744:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761708.356791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274176000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761708.356830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761708.356836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761708.356841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761708.356847:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 753 00000100:00080000:2.0:1787761708.356851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761708.356856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761708.356868:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761708.356881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274176128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761708.356991:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176000:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761708.357102:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761708.357168:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761708/real 1787761708] req@ffff9ac4523bc3c0 x1874603274176000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761708.357202:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274176000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761708.357216:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761708.357221:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761708.357227:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176000:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761708.357237:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176128:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761708.357269:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761708.357283:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761708/real 1787761708] req@ffff9ac4523be940 x1874603274176128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761708.357299:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274176128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761708.357309:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761708.357313:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761708.357317:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176128:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761708.932736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761709.380640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761709.956839:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761710.404664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761710.980659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761711.428618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761712.004680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761712.452692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761713.028651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761713.476598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761713.476622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761713.476627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761713.476631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761713.476640:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 758 00000100:00080000:1.0:1787761713.476649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761713.476655:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761713.476688:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274176256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761713.476724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761713.476727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761713.476728:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761713.476731:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 758 00000100:00080000:1.0:1787761713.476734:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761713.476737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761713.476745:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761713.476755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274176384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761713.476784:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176256:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761713.476845:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761713.476878:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761713/real 1787761713] req@ffff9ac447843840 x1874603274176256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761713.476892:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274176256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761713.476900:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761713.476903:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761713.476906:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176256:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761713.476910:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176384:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761713.476939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761713.476950:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761713/real 1787761713] req@ffff9ac447841e00 x1874603274176384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761713.476967:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274176384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761713.476973:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761713.476975:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761713.476977:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176384:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761714.052749:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761714.500595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761715.076644:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761715.524602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761716.100721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761716.548598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761717.124794:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761717.572611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761718.148700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761718.596644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761718.596644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761718.596653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761718.596657:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761718.596666:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 763 00000100:00080000:0.0:1787761718.596674:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761718.596680:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761718.596712:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274176512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761718.596735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761718.596739:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761718.596741:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761718.596746:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 763 00000100:00080000:0.0:1787761718.596751:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761718.596756:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761718.596767:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761718.596769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176512:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761718.596786:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274176640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761718.596827:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761718.596861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761718/real 1787761718] req@ffff9ac451f861c0 x1874603274176512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761718.596874:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274176512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761718.596882:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761718.596885:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761718.596903:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176512:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761718.596922:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176640:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761718.596939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761718.596947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761718/real 1787761718] req@ffff9ac451f84780 x1874603274176640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761718.596963:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274176640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761718.596969:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761718.596971:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761718.596973:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176640:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761719.172742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761719.620663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761720.196760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761720.644635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761721.220722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761721.668628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761722.244719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761722.692597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761723.268718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761723.716647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761723.716682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761723.716692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761723.716698:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761723.716710:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 768 00000100:00080000:2.0:1787761723.716724:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761723.716732:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761723.716781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274176768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761723.716813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761723.716817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761723.716819:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761723.716822:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 768 00000100:00080000:2.0:1787761723.716825:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761723.716828:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761723.716840:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761723.716850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274176896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761723.716883:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176768:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761723.716962:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761723.717014:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761723/real 1787761723] req@ffff9ac4523bda40 x1874603274176768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761723.717042:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274176768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761723.717058:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761723.717064:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761723.717070:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274176768:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761723.717101:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176896:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761723.717121:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761723.717132:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761723/real 1787761723] req@ffff9ac4523bf840 x1874603274176896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761723.717141:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274176896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761723.717165:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761723.717168:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761723.717170:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274176896:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761724.292759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761724.740632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761725.316738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761725.764671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761726.340743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761726.788621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761727.364739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761727.812673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761728.388801:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787761728.836638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761728.836654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761728.836657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761728.836670:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761728.836683:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 773 00000100:00080000:2.0:1787761728.836708:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761728.836718:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761728.836785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274177024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761728.836844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761728.836852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761728.836855:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761728.836873:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 773 00000100:00080000:2.0:1787761728.836880:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761728.836886:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761728.836898:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761728.836918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274177152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761728.836954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177024:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761728.837046:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761728.837126:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761728/real 1787761728] req@ffff9ac4523bf840 x1874603274177024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761728.837155:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274177024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761728.837169:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761728.837175:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761728.837180:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177024:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761728.837191:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177152:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761728.837223:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761728.837239:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761728/real 1787761728] req@ffff9ac4523bda40 x1874603274177152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761728.837254:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274177152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761728.837265:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761728.837270:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761728.837274:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177152:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761729.412788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761729.860647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761730.436791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761730.884624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761731.460742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761731.908621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761732.484748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761732.932635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761733.508834:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761733.956614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761733.956675:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761733.956692:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761733.956700:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761733.956717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 778 00000100:00080000:2.0:1787761733.956734:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761733.956760:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761733.956843:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274177280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761733.956895:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761733.956902:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761733.956906:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761733.956915:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 778 00000100:00080000:2.0:1787761733.956925:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761733.956957:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761733.956959:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177280:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787761733.956981:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761733.957008:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274177408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761733.957126:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177408:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761734.532667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761734.980669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761735.556721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761736.004658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761736.580676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761737.028612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787761737.028843:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787761737.028889:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787761737.028910:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787761737.028954:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787761737.029018:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761733/real 1787761737] req@ffff9ac4523bc3c0 x1874603274177408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761737.029049:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274177408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761737.029065:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761737.029071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761737.029076:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177408:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787761737.029086:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761733/real 1787761737] req@ffff9ac4523be940 x1874603274177280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761737.029101:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274177280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761737.029108:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761737.029112:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761737.029116:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177280:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787761737.604674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761738.052605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761738.628705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761739.076720:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761739.076739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761739.076774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761739.076781:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761739.076796:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 783 00000100:00080000:1.0:1787761739.076814:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761739.076850:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761739.076907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274177536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761739.076958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761739.076964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761739.076967:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761739.076973:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 783 00000100:00080000:1.0:1787761739.076979:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761739.076984:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761739.076999:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761739.077018:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274177664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761739.077036:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177536:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761739.077186:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761739.077247:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761739/real 1787761739] req@ffff9ac447841a40 x1874603274177536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761739.077273:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274177536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761739.077295:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761739.077302:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761739.077308:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177536:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761739.077343:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177664:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761739.077376:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761739.077395:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761739/real 1787761739] req@ffff9ac4478412c0 x1874603274177664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761739.077412:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274177664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761739.077424:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761739.077429:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761739.077434:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177664:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761739.652772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761740.100664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761740.676759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761741.124606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761741.701052:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761742.148629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761742.724761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761743.172598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761743.748665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761744.196616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761744.196631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761744.196643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761744.196649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761744.196664:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 788 00000100:00080000:1.0:1787761744.196681:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761744.196693:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761744.196793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274177792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761744.196825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761744.196846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761744.196849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761744.196856:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 788 00000100:00080000:1.0:1787761744.196896:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761744.196906:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761744.196922:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761744.196924:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761744.196953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274177920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761744.197032:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761744.197115:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761744/real 1787761744] req@ffff9ac447843480 x1874603274177792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761744.197129:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274177792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761744.197138:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761744.197142:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761744.197145:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274177792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761744.197167:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177920:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761744.197184:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761744.197193:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761744/real 1787761744] req@ffff9ac4478412c0 x1874603274177920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761744.197201:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274177920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761744.197209:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761744.197212:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761744.197216:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274177920:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761744.772693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761745.220651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761745.796912:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761746.244660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761746.821053:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761747.268631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761747.844801:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761748.292656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761748.868755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761749.316710:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761749.316740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761749.316751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761749.316757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761749.316772:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 794 00000100:00080000:2.0:1787761749.316786:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761749.316795:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761749.316844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274178048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761749.316883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761749.316888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761749.316891:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761749.316896:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 794 00000100:00080000:2.0:1787761749.316901:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761749.316905:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761749.316917:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761749.316935:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274178176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761749.316975:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178048:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761749.317069:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761749.317138:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761749/real 1787761749] req@ffff9ac4523bc000 x1874603274178048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761749.317163:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274178048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761749.317197:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761749.317203:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761749.317208:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178048:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761749.317237:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178176:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761749.317271:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761749.317295:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761749/real 1787761749] req@ffff9ac4523bd680 x1874603274178176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761749.317320:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274178176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761749.317337:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761749.317344:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761749.317351:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178176:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761749.892720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761750.340665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761750.916714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761751.364590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761751.940704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761752.388636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761752.964764:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761753.412681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761753.988756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761754.436677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761754.436738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761754.436747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761754.436753:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761754.436765:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 799 00000100:00080000:0.0:1787761754.436778:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761754.436788:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761754.436833:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274178304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761754.436874:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761754.436898:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761754.436901:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761754.436907:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 799 00000100:00080000:0.0:1787761754.436913:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761754.436920:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761754.436934:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761754.436954:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274178432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761754.436963:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178304:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761754.437059:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761754.437125:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761754/real 1787761754] req@ffff9ac451f84000 x1874603274178304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761754.437148:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274178304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761754.437162:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761754.437168:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761754.437173:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178304:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761754.437201:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178432:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761754.437230:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761754.437246:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761754/real 1787761754] req@ffff9ac451f870c0 x1874603274178432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761754.437262:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274178432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761754.437272:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761754.437276:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761754.437294:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178432:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761755.012749:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761755.460656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761756.036761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761756.484676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761757.060729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761757.508689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761758.084767:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761758.532698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761759.108760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761759.556601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761759.556602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761759.556623:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761759.556627:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761759.556635:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 804 00000100:00080000:0.0:1787761759.556655:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761759.556662:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761759.556694:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274178560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761759.556718:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761759.556722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761759.556724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761759.556728:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 804 00000100:00080000:0.0:1787761759.556732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761759.556736:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761759.556746:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761759.556759:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274178688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761759.556773:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178560:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761759.556834:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761759.556878:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761759/real 1787761759] req@ffff9ac451f86940 x1874603274178560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761759.556897:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274178560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761759.556908:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761759.556913:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761759.556917:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178560:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761759.556951:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178688:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761759.556971:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761759.556980:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761759/real 1787761759] req@ffff9ac451f87c00 x1874603274178688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761759.556989:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274178688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761759.556995:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761759.556997:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761759.557011:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178688:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761760.132656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761760.580609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761761.156745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761761.604693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761762.180675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761762.628603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761763.204770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761763.652663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761764.228681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761764.676620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761764.676651:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761764.676660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761764.676665:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761764.676677:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 809 00000100:00080000:2.0:1787761764.676692:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761764.676701:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761764.676749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274178816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761764.676777:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761764.676781:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761764.676783:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761764.676787:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 809 00000100:00080000:2.0:1787761764.676790:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761764.676793:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761764.676804:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761764.676815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274178944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761764.676842:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178816:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761764.676924:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761764.676993:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761764/real 1787761764] req@ffff9ac4523bc780 x1874603274178816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761764.677010:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274178816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761764.677021:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761764.677025:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761764.677029:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274178816:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761764.677036:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178944:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761764.677056:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761764.677066:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761764/real 1787761764] req@ffff9ac4523be580 x1874603274178944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761764.677076:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274178944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761764.677082:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761764.677084:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761764.677087:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274178944:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761765.252773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761765.700615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761766.276654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761766.724629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761767.300668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761767.748605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761768.324655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761768.772623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761769.348647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761769.796613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761769.796640:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761769.796652:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761769.796657:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761769.796668:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 814 00000100:00080000:0.0:1787761769.796679:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761769.796688:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761769.796729:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274179072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761769.796755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761769.796758:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761769.796759:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761769.796763:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 814 00000100:00080000:0.0:1787761769.796765:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761769.796768:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761769.796778:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761769.796789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274179200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761769.796807:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179072:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761769.796889:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761769.796937:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761769/real 1787761769] req@ffff9ac451f84b40 x1874603274179072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761769.796953:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274179072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761769.796974:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761769.796977:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761769.796981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179072:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761769.797010:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179200:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761769.797037:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761769.797050:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761769/real 1787761769] req@ffff9ac451f87c00 x1874603274179200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761769.797064:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274179200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761769.797073:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761769.797077:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761769.797081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179200:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761770.372665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761770.820651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761771.396638:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761771.844614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761772.420640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761772.868634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761773.444707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761773.892621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761774.468690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761774.916725:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761774.916761:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761774.916772:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761774.916788:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761774.916807:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 819 00000100:00080000:0.0:1787761774.916820:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761774.916832:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761774.916898:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f86940 x1874603274179328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761774.916932:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761774.916937:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761774.916939:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761774.916944:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 819 00000100:00080000:0.0:1787761774.916950:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761774.916957:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761774.916971:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761774.916988:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179328:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761774.916992:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274179456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761774.917108:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761774.917189:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761774/real 1787761774] req@ffff9ac451f86940 x1874603274179328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761774.917217:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f86940 x1874603274179328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761774.917232:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761774.917237:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761774.917243:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f86940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179328:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761774.917289:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179456:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761774.917323:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761774.917340:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761774/real 1787761774] req@ffff9ac451f870c0 x1874603274179456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761774.917355:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274179456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761774.917366:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761774.917370:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761774.917374:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179456:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761775.492699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761775.940650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761776.516709:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761776.964657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761777.540687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761777.988656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761778.564725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761779.012662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761779.588709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761780.036646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761780.036682:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761780.036696:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761780.036702:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761780.036715:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 824 00000100:00080000:2.0:1787761780.036729:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761780.036739:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761780.036807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274179584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761780.036850:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761780.036855:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761780.036858:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761780.036863:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 824 00000100:00080000:2.0:1787761780.036869:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761780.036874:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761780.036887:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761780.036905:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274179712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761780.036938:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179584:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761780.037040:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761780.037111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761780/real 1787761780] req@ffff9ac4523bcf00 x1874603274179584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761780.037133:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274179584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761780.037144:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761780.037148:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761780.037151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761780.037179:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179712:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761780.037199:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761780.037209:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761780/real 1787761780] req@ffff9ac4523bcb40 x1874603274179712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761780.037241:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274179712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761780.037248:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761780.037250:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761780.037253:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179712:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761780.612744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761781.060664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761781.636763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761782.084651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761782.660669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761783.108590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761783.684664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761784.132620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761784.708691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761785.156599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761785.156671:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761785.156678:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761785.156683:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761785.156694:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 829 00000100:00080000:1.0:1787761785.156707:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761785.156715:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761785.156762:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274179840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761785.156793:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761785.156796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761785.156797:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761785.156801:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 829 00000100:00080000:1.0:1787761785.156803:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761785.156806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761785.156817:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761785.156827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274179968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761785.156836:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179840:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761785.156915:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761785.156957:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761785/real 1787761785] req@ffff9ac447841a40 x1874603274179840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761785.156980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274179840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761785.156990:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761785.156994:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761785.156997:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274179840:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761785.157020:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179968:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761785.157036:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761785.157058:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761785/real 1787761785] req@ffff9ac447841e00 x1874603274179968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761785.157071:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274179968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761785.157081:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761785.157084:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761785.157088:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274179968:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761785.732666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761786.180591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761786.756643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761787.204581:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761787.780739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761788.228638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761788.804688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761789.252665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761789.828712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761790.276690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761790.276691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761790.276707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761790.276714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761790.276727:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 835 00000100:00080000:1.0:1787761790.276740:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761790.276750:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761790.276796:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274180096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761790.276833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761790.276838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761790.276840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761790.276845:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 835 00000100:00080000:1.0:1787761790.276851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761790.276856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761790.276868:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761790.276886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274180224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761790.276912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180096:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761790.277014:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761790.277071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761790/real 1787761790] req@ffff9ac447843840 x1874603274180096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761790.277088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274180096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761790.277098:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761790.277102:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761790.277105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180096:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761790.277133:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180224:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761790.277159:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761790.277174:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761790/real 1787761790] req@ffff9ac447842580 x1874603274180224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761790.277188:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274180224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761790.277222:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761790.277226:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761790.277230:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180224:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761790.852748:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761791.300666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761791.876694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761792.324677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761792.900673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761793.348666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761793.924687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761794.372664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761794.948744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761795.396683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761795.396686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761795.396699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761795.396705:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761795.396717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 840 00000100:00080000:1.0:1787761795.396730:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761795.396740:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761795.396801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274180352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761795.396839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761795.396844:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761795.396847:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761795.396853:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 840 00000100:00080000:1.0:1787761795.396859:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761795.396866:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761795.396878:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761795.396897:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274180480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761795.396902:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180352:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761795.397016:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761795.397066:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761795/real 1787761795] req@ffff9ac447840000 x1874603274180352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761795.397083:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274180352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761795.397093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761795.397097:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761795.397100:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180352:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761795.397127:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180480:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761795.397143:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761795.397152:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761795/real 1787761795] req@ffff9ac447840f00 x1874603274180480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761795.397160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274180480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761795.397166:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761795.397169:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761795.397171:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180480:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761795.972719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761796.420653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761796.996718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761797.444647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761798.020778:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761798.468649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761799.044745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761799.492654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761800.068809:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761800.516644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761800.516657:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761800.516664:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761800.516678:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 845 00000100:00100000:3.0:1787761800.516684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761800.516692:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761800.516705:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761800.516752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274180608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761800.516789:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761800.516794:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761800.516797:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761800.516802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 845 00000100:00080000:1.0:1787761800.516807:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761800.516827:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761800.516840:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761800.516870:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274180736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761800.516919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761800.517044:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180736:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761801.092713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761801.540659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761802.116768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761802.564650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761803.140733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761803.588617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761803.588780:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787761803.588834:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761803.588866:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761803.588927:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787761803.589051:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761800/real 1787761803] req@ffff9ac447840780 x1874603274180608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761803.589114:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274180608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761803.589137:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761803.589147:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761803.589154:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761803.589193:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761800/real 1787761803] req@ffff9ac447841680 x1874603274180736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761803.589219:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274180736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761803.589237:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761803.589244:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761803.589251:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180736:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761804.164686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761804.612609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761805.188651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761805.636583:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761805.636619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761805.636625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761805.636631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761805.636641:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 850 00000100:00080000:0.0:1787761805.636655:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761805.636680:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761805.636722:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274180864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761805.636750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761805.636754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761805.636755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761805.636759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 850 00000100:00080000:0.0:1787761805.636763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761805.636766:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761805.636777:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761805.636788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274180992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761805.636793:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180864:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761805.636897:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761805.636956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761805/real 1787761805] req@ffff9ac451f84000 x1874603274180864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761805.636980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274180864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761805.636993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761805.636997:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761805.637003:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274180864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761805.637035:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180992:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761805.637082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761805.637098:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761805/real 1787761805] req@ffff9ac451f84780 x1874603274180992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761805.637111:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274180992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761805.637129:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761805.637133:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761805.637136:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274180992:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761806.212633:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761806.660649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761807.236712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761807.684632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761808.261029:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761808.708657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761809.284884:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761809.732643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761810.308832:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761810.756634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761810.756668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761810.756679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761810.756685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761810.756697:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 855 00000100:00080000:2.0:1787761810.756711:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761810.756720:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761810.756767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274181120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761810.756804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761810.756809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761810.756812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761810.756817:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 855 00000100:00080000:2.0:1787761810.756821:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761810.756843:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761810.756856:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761810.756875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274181248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761810.756937:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181120:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761810.757021:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761810.757070:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761810/real 1787761810] req@ffff9ac4523be1c0 x1874603274181120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761810.757093:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274181120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761810.757107:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761810.757112:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761810.757117:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761810.757126:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274181248:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761810.757151:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761810.757165:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761810/real 1787761810] req@ffff9ac4523bf0c0 x1874603274181248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761810.757180:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274181248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761810.757191:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761810.757207:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761810.757212:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274181248:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761811.332701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761811.780610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761812.356630:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761812.804583:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761813.380677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761813.828635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761814.404647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761814.852594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761815.428737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761815.876714:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761815.876748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761815.876759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761815.876766:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761815.876779:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 860 00000100:00080000:0.0:1787761815.876793:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761815.876802:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761815.876847:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274181376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761815.876887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761815.876909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761815.876912:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761815.876919:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 860 00000100:00080000:0.0:1787761815.876925:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761815.876932:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761815.876946:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761815.876958:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181376:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761815.876969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274181504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761815.877047:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761815.877110:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761815/real 1787761815] req@ffff9ac451f861c0 x1874603274181376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761815.877140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274181376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761815.877153:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761815.877158:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761815.877164:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181376:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761815.877192:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274181504:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761815.877242:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761815.877262:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761815/real 1787761815] req@ffff9ac451f85680 x1874603274181504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761815.877278:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274181504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761815.877289:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761815.877293:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761815.877297:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274181504:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761816.452727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761816.900672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761817.476753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761817.924623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761818.500771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761818.948654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761819.524773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761819.972621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761820.548715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761820.996661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761820.996730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761820.996743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761820.996750:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761820.996765:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 865 00000100:00080000:2.0:1787761820.996779:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761820.996788:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761820.996837:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274181632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761820.996877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761820.996882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761820.996884:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761820.996889:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 865 00000100:00080000:2.0:1787761820.996894:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761820.996898:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761820.996911:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761820.996929:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274181760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761820.996952:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181632:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761820.997038:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761820.997106:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761820/real 1787761820] req@ffff9ac4523bf0c0 x1874603274181632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761820.997160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274181632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761820.997201:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761820.997208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761820.997213:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181632:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761820.997248:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274181760:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761820.997283:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761820.997300:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761820/real 1787761820] req@ffff9ac4523be1c0 x1874603274181760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761820.997316:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274181760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761820.997327:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761820.997331:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761820.997335:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274181760:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761821.572784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761822.020614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761822.596730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761823.044591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761823.620746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761824.068650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787761824.644724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761825.092607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761825.668688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761826.116670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761826.116674:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761826.116695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761826.116700:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761826.116730:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 870 00000100:00080000:1.0:1787761826.116743:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761826.116753:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761826.116803:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274181888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761826.116847:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761826.116852:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761826.116855:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761826.116860:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 870 00000100:00080000:1.0:1787761826.116864:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761826.116869:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761826.116880:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761826.116897:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274182016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761826.116946:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181888:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761826.117053:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761826.117116:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761826/real 1787761826] req@ffff9ac4478403c0 x1874603274181888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761826.117140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274181888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761826.117179:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761826.117197:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761826.117203:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274181888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761826.117213:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182016:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761826.117248:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761826.117287:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761826/real 1787761826] req@ffff9ac4478421c0 x1874603274182016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761826.117313:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274182016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761826.117330:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761826.117336:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761826.117344:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182016:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787761826.692737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761827.140646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761827.716722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761828.164627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761828.740704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761829.188637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761829.764686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761830.212605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761830.788678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761831.236617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761831.236626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1787761831.236626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761831.236631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761831.236641:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 875 00000100:00080000:0.0:1787761831.236650:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761831.236656:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761831.236690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274182144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761831.236715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761831.236718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761831.236720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761831.236723:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 875 00000100:00080000:0.0:1787761831.236725:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761831.236727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761831.236736:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761831.236761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274182272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761831.236766:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182144:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761831.236827:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761831.236862:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761831/real 1787761831] req@ffff9ac451f843c0 x1874603274182144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761831.236875:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274182144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761831.236889:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761831.236892:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761831.236895:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182144:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761831.236913:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182272:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761831.236930:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761831.236938:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761831/real 1787761831] req@ffff9ac451f85680 x1874603274182272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761831.236946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274182272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761831.236952:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761831.236954:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761831.236956:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182272:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761831.812654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761832.260617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761832.836652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761833.284627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761833.860747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761834.308627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761834.884746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761835.332643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761835.908660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761836.356599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761836.356627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761836.356635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761836.356656:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761836.356666:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 881 00000100:00080000:2.0:1787761836.356676:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761836.356682:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761836.356712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274182400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761836.356734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761836.356737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761836.356738:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761836.356741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 881 00000100:00080000:2.0:1787761836.356743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761836.356746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761836.356753:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761836.356763:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274182528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761836.356785:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182400:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761836.356839:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761836.356873:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761836/real 1787761836] req@ffff9ac4523bcb40 x1874603274182400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761836.356886:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274182400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761836.356894:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761836.356896:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761836.356899:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182400:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761836.356911:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182528:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761836.356928:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761836.356936:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761836/real 1787761836] req@ffff9ac4523bcf00 x1874603274182528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761836.356945:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274182528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761836.356950:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761836.356952:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761836.356955:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182528:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761836.932663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761837.380655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761837.956655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761838.404597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761838.980720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761839.428611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761840.004775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761840.452644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761841.028711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761841.476635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761841.476731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761841.476740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761841.476746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761841.476759:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 886 00000100:00080000:0.0:1787761841.476772:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761841.476782:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761841.476829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274182656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761841.476868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761841.476873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761841.476875:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761841.476881:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 886 00000100:00080000:0.0:1787761841.476885:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761841.476889:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761841.476902:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761841.476917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274182784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761841.476964:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182656:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761841.477096:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761841.477148:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761841/real 1787761841] req@ffff9ac451f861c0 x1874603274182656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761841.477170:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274182656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761841.477183:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761841.477189:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761841.477193:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182656:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761841.477202:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182784:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761841.477230:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761841.477243:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761841/real 1787761841] req@ffff9ac451f84780 x1874603274182784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761841.477291:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274182784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761841.477303:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761841.477307:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761841.477311:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274182784:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761842.052654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761842.500654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761843.076654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761843.524613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761844.100820:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761844.548633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761845.124675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761845.572604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761846.148668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761846.596651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761846.596663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761846.596670:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787761846.596673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761846.596685:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 891 00000100:00080000:0.0:1787761846.596701:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761846.596711:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761846.596758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274182912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761846.596793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761846.596798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761846.596800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761846.596805:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 891 00000100:00080000:0.0:1787761846.596809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761846.596814:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761846.596826:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761846.596842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274183040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761846.596876:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182912:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761846.596960:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761846.597035:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761846/real 1787761846] req@ffff9ac451f84000 x1874603274182912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761846.597061:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274182912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761846.597075:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761846.597080:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761846.597086:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274182912:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761846.597116:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183040:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761846.597147:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761846.597163:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761846/real 1787761846] req@ffff9ac451f870c0 x1874603274183040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761846.597179:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274183040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761846.597214:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761846.597219:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761846.597224:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183040:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761847.172768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761847.620593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761848.196736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761848.644603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761849.220734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761849.668647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761850.244790:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761850.692634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761851.268773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761851.716647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761851.716697:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761851.716707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761851.716713:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761851.716726:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 896 00000100:00080000:1.0:1787761851.716739:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761851.716748:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761851.716795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274183168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761851.716831:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761851.716837:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761851.716839:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761851.716845:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 896 00000100:00080000:1.0:1787761851.716849:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761851.716854:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761851.716867:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761851.716885:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274183296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761851.716899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274183168:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761851.717017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761851.717068:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761851/real 1787761851] req@ffff9ac4478430c0 x1874603274183168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761851.717118:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274183168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761851.717132:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761851.717138:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761851.717144:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274183168:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761851.717173:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183296:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761851.717204:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761851.717222:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761851/real 1787761851] req@ffff9ac447842d00 x1874603274183296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761851.717248:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274183296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761851.717264:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761851.717270:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761851.717277:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183296:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761852.292789:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761852.740655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761853.316777:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761853.764662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761854.340747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761854.788609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761855.364755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761855.812677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761856.388728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761856.836647:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787761856.836651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761856.836660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761856.836692:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761856.836704:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 901 00000100:00080000:1.0:1787761856.836717:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761856.836726:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761856.836770:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274183424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761856.836806:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761856.836810:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761856.836811:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761856.836814:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 901 00000100:00080000:1.0:1787761856.836817:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761856.836820:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761856.836830:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761856.836841:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274183552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761856.836895:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274183424:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761856.836988:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761856.837035:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761856/real 1787761856] req@ffff9ac447842940 x1874603274183424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761856.837063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274183424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761856.837072:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761856.837075:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761856.837079:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274183424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761856.837084:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183552:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761856.837102:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761856.837111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761856/real 1787761856] req@ffff9ac447843c00 x1874603274183552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761856.837119:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274183552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761856.837125:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761856.837128:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761856.837130:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183552:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761857.412663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761857.860615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761858.436651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761858.884616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761859.460649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787761859.908671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787761860.484752:0:9840:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1787761860.484834:0:5438:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787761860.484854:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761860.484856:0:5438:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787761860.484978:0:5438:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787761860.484991:0:5438:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787761860.485045:0:5438:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9ac4523c1000 lock: ffff9ac4414a8800/0x88466b5949dc2ecc lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2dffe29982e24718 expref: -99 pid: 9835 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787761860.485076:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761860.485080:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787761860.485083:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761860.485131:0:2346:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9ac451f86940 x1874603274183680/t0(0) o103->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787761860.932694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761861.508715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761861.956692:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761861.956705:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761861.956710:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761861.956724:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 906 00000100:00100000:1.0:1787761861.956740:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761861.956742:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761861.956758:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761861.956823:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274183808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761861.956859:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761861.956863:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761861.956864:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761861.956869:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 906 00000100:00080000:3.0:1787761861.956872:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761861.956896:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761861.956928:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787761861.956945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274183936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761861.956969:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274183808:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761861.957064:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761861.957117:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761861/real 1787761861] req@ffff9ac451c50f00 x1874603274183808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761861.957148:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274183808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761861.957170:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761861.957178:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761861.957187:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274183808:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761861.957230:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183936:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761861.957271:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761861.957290:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761861/real 1787761861] req@ffff9ac451c52580 x1874603274183936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761861.957307:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274183936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761861.957318:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761861.957339:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761861.957344:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274183936:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761862.532744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761862.980639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761863.556779:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761864.004700:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761864.580714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761865.028647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761865.604728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761866.052673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761866.628757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761867.076670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761867.076798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761867.076808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761867.076813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761867.076825:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 911 00000100:00080000:0.0:1787761867.076840:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761867.076849:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761867.076934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274184064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761867.076976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761867.076982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761867.076984:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761867.076990:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 911 00000100:00080000:0.0:1787761867.076995:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761867.077002:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761867.077016:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761867.077033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274184192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761867.077039:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761867.077183:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184192:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761867.652776:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761868.100646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787761868.676780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761869.124738:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761869.700715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761870.148906:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787761870.149057:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787761870.149158:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787761870.149186:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787761870.149250:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787761870.149333:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761867/real 1787761870] req@ffff9ac451f84b40 x1874603274184192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761870.149409:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274184192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761870.149430:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761870.149439:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761870.149447:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184192:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787761870.149461:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761867/real 1787761870] req@ffff9ac451f87c00 x1874603274184064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761870.149485:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274184064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761870.149499:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761870.149505:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761870.149510:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184064:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787761870.724710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761871.172692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761871.748761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761872.196601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761872.196609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761872.196616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761872.196619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761872.196629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 916 00000100:00080000:0.0:1787761872.196637:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761872.196644:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761872.196673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274184320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761872.196696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761872.196707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761872.196709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761872.196713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 916 00000100:00080000:0.0:1787761872.196716:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761872.196720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761872.196724:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184320:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787761872.196729:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761872.196742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274184448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761872.196788:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761872.196838:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761872/real 1787761872] req@ffff9ac451f87c00 x1874603274184320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761872.196857:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274184320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761872.196873:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761872.196877:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761872.196881:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761872.196905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184448:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761872.196929:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761872.196940:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761872/real 1787761872] req@ffff9ac451f84b40 x1874603274184448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761872.196949:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274184448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761872.196954:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761872.196957:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761872.196959:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184448:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787761872.772715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761873.220644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761873.796743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761874.244665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761874.820883:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761875.268642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761875.844664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761876.292655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761876.868680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787761877.316654:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761877.316665:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787761877.316678:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761877.316690:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 922 00000100:00080000:3.0:1787761877.316705:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787761877.316715:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787761877.316715:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787761877.316769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274184576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787761877.316810:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787761877.316814:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787761877.316815:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787761877.316819:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 922 00000100:00080000:3.0:1787761877.316822:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787761877.316845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787761877.316857:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761877.316878:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787761877.316878:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274184704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761877.316985:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761877.317037:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761877/real 1787761877] req@ffff9ac451c503c0 x1874603274184576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761877.317060:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274184576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761877.317076:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761877.317081:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761877.317087:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761877.317118:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184704:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761877.317144:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761877.317159:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761877/real 1787761877] req@ffff9ac451c50780 x1874603274184704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761877.317183:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274184704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761877.317200:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761877.317206:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761877.317213:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184704:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761877.892719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761878.340702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761878.916728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761879.364695:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761879.940757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761880.388664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761880.964793:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761881.412677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761881.988721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761882.436682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761882.436719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761882.436730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761882.436737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761882.436749:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 927 00000100:00080000:2.0:1787761882.436784:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761882.436794:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761882.436842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274184832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761882.436898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761882.436903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761882.436905:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761882.436911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 927 00000100:00080000:2.0:1787761882.436917:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761882.436925:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761882.436939:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761882.436956:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761882.436965:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274184960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761882.437047:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761882.437098:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761882/real 1787761882] req@ffff9ac4523be580 x1874603274184832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761882.437121:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274184832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761882.437136:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761882.437141:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761882.437147:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274184832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761882.437193:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184960:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761882.437223:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761882.437238:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761882/real 1787761882] req@ffff9ac4523bc780 x1874603274184960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761882.437255:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274184960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761882.437266:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761882.437271:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761882.437275:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274184960:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761883.012733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761883.460627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761884.036774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761884.484682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761885.060738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761885.508681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761886.084723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761886.532665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761887.108846:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761887.556692:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787761887.556698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761887.556717:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761887.556744:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761887.556758:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 932 00000100:00080000:1.0:1787761887.556772:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761887.556781:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761887.556827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274185088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761887.556863:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761887.556869:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761887.556871:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761887.556877:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 932 00000100:00080000:1.0:1787761887.556882:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761887.556888:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761887.556901:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761887.556921:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274185216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761887.556934:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185088:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761887.557020:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761887.557071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761887/real 1787761887] req@ffff9ac447840b40 x1874603274185088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761887.557096:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274185088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761887.557110:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761887.557115:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761887.557121:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761887.557150:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185216:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761887.557177:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761887.557192:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761887/real 1787761887] req@ffff9ac447843c00 x1874603274185216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761887.557252:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274185216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761887.557271:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761887.557278:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761887.557286:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185216:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761888.132800:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761888.580618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761889.156691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761889.604606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761890.180730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761890.628597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761891.204673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761891.652659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761892.228713:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761892.676642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787761892.676642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761892.676653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761892.676658:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761892.676667:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 937 00000100:00080000:0.0:1787761892.676678:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761892.676685:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761892.676745:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274185344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761892.676773:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761892.676777:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761892.676779:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761892.676785:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 937 00000100:00080000:0.0:1787761892.676791:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761892.676799:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761892.676811:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761892.676812:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761892.676840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274185472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761892.676902:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761892.676941:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761892/real 1787761892] req@ffff9ac451f84b40 x1874603274185344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761892.676959:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274185344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761892.676969:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761892.676973:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761892.676977:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761892.676998:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185472:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761892.677017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761892.677028:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761892/real 1787761892] req@ffff9ac451f87c00 x1874603274185472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761892.677039:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274185472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761892.677046:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761892.677049:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761892.677052:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761893.252776:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761893.700717:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761894.276746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761894.724642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761895.300755:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761895.748705:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761896.324737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761896.772679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761897.348736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761897.796755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761897.796767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787761897.796780:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761897.796799:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761897.796815:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 942 00000100:00080000:1.0:1787761897.796829:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761897.796839:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761897.796886:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274185600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761897.796937:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761897.796945:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761897.796949:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761897.796959:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 942 00000100:00080000:1.0:1787761897.796967:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761897.796977:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761897.796994:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761897.797005:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761897.797032:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274185728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761897.797096:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761897.797150:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761897/real 1787761897] req@ffff9ac447842940 x1874603274185600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761897.797174:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274185600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761897.797188:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761897.797194:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761897.797199:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761897.797227:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185728:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761897.797253:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761897.797267:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761897/real 1787761897] req@ffff9ac447842d00 x1874603274185728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761897.797282:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274185728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761897.797293:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761897.797297:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761897.797302:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185728:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761898.372682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761898.820609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761899.396690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761899.844677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761900.420828:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761900.868694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761901.444733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761901.892763:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761902.468745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761902.916637:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761902.916644:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761902.916648:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761902.916674:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 947 00000100:00080000:1.0:1787761902.916683:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787761902.916685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761902.916690:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761902.916724:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274185856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761902.916743:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761902.916758:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761902.916759:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761902.916763:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 947 00000100:00080000:1.0:1787761902.916765:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761902.916768:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787761902.916776:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761902.916787:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274185984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787761902.916800:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185856:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787761902.916856:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761902.916898:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761902/real 1787761902] req@ffff9ac4478430c0 x1874603274185856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761902.916911:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274185856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761902.916919:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761902.916922:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761902.916924:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274185856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761902.916941:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185984:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761902.916956:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761902.916964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761902/real 1787761902] req@ffff9ac4478421c0 x1874603274185984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761902.916972:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274185984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761902.916977:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761902.916979:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761902.916981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274185984:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761903.492719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761903.940641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761904.516798:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761904.964673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761905.540788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761905.988717:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761906.564676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761907.012605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761907.588706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787761908.036629:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761908.036637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787761908.036640:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787761908.036645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787761908.036650:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 952 00000100:00080000:1.0:1787761908.036661:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761908.036668:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761908.036702:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274186112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787761908.036720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787761908.036722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787761908.036724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787761908.036727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 952 00000100:00080000:1.0:1787761908.036729:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787761908.036732:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761908.036754:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186112:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787761908.036756:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761908.036770:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274186240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787761908.036816:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761908.036851:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761908/real 1787761908] req@ffff9ac4478403c0 x1874603274186112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761908.036864:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274186112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761908.036883:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761908.036886:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761908.036889:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186112:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761908.036907:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274186240:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787761908.036924:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787761908.036933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761908/real 1787761908] req@ffff9ac447841680 x1874603274186240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787761908.036941:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274186240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787761908.036954:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787761908.036958:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787761908.036961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274186240:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761908.612685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761909.060619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761909.636677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761910.084627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761910.660687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761911.108624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761911.684651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761912.132675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761912.708662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761913.156621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761913.156638:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761913.156644:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761913.156648:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761913.156657:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 957 00000100:00080000:0.0:1787761913.156666:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761913.156672:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761913.156705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274186368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761913.156735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761913.156751:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761913.156753:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761913.156757:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 957 00000100:00080000:0.0:1787761913.156761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761913.156765:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761913.156773:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761913.156775:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761913.156786:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274186496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761913.156835:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761913.156876:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761913/real 1787761913] req@ffff9ac451f870c0 x1874603274186368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761913.156891:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274186368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761913.156898:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761913.156901:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761913.156904:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761913.156923:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274186496:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761913.156938:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761913.156946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761913/real 1787761913] req@ffff9ac451f84000 x1874603274186496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761913.156954:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274186496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761913.156960:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761913.156963:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761913.156972:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274186496:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761913.732659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761914.180588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761914.756653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761915.204611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761915.780657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761916.228603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761916.804750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761917.252645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761917.828730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761918.276588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761918.276601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761918.276607:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761918.276623:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761918.276632:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 963 00000100:00080000:0.0:1787761918.276641:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761918.276647:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761918.276677:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274186624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761918.276698:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761918.276701:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761918.276702:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761918.276705:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 963 00000100:00080000:0.0:1787761918.276707:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761918.276709:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761918.276728:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761918.276738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274186752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761918.276744:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186624:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761918.276827:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761918.276858:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761918/real 1787761918] req@ffff9ac451f84780 x1874603274186624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761918.276871:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274186624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761918.276880:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761918.276884:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761918.276888:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761918.276911:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274186752:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761918.276937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761918.276955:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761918/real 1787761918] req@ffff9ac451f861c0 x1874603274186752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761918.276964:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274186752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761918.276970:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761918.276973:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761918.276981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274186752:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761918.852650:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761919.300667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761919.876762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761920.324636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761920.900723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761921.348665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761921.924731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761922.372664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761922.948756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761923.396664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761923.396698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761923.396709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761923.396737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761923.396750:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 968 00000100:00080000:2.0:1787761923.396763:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761923.396773:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761923.396818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274186880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761923.396859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761923.396864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761923.396867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761923.396872:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 968 00000100:00080000:2.0:1787761923.396876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761923.396881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761923.396895:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761923.396915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274187008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761923.396922:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186880:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761923.397015:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761923.397080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761923/real 1787761923] req@ffff9ac4523bc000 x1874603274186880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761923.397109:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274186880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761923.397125:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761923.397132:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761923.397138:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274186880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761923.397166:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187008:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761923.397195:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761923.397209:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761923/real 1787761923] req@ffff9ac4523be940 x1874603274187008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761923.397224:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274187008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761923.397234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761923.397238:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761923.397260:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187008:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761923.972704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761924.420702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761924.996734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761925.444637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761926.020743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761926.468631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761927.044730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761927.492627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761928.068705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787761928.516622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761928.516635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761928.516641:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761928.516655:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 973 00000100:00100000:1.0:1787761928.516666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761928.516669:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761928.516685:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761928.516733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274187136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761928.516768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761928.516790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761928.516793:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761928.516801:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 973 00000100:00080000:2.0:1787761928.516808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761928.516815:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787761928.516830:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187136:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787761928.516830:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761928.516856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274187264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761928.516922:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761928.516972:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761928/real 1787761928] req@ffff9ac4523bc3c0 x1874603274187136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761928.516995:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274187136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761928.517009:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761928.517014:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761928.517019:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761928.517046:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187264:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761928.517072:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761928.517087:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761928/real 1787761928] req@ffff9ac4523bda40 x1874603274187264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761928.517102:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274187264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761928.517113:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761928.517118:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761928.517134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187264:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761929.092759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761929.540637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761930.116715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761930.564623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761931.140728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761931.588686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761932.164684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761932.612656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761933.188731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761933.636601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761933.636647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761933.636656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761933.636661:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761933.636671:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 978 00000100:00080000:2.0:1787761933.636702:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761933.636713:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761933.636769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274187392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761933.636821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761933.636829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761933.636834:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761933.636845:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 978 00000100:00100000:1.0:1787761933.636854:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187392:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787761933.636854:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761933.636875:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761933.636896:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761933.636926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274187520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761933.636983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187520:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761934.212765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761934.660661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761935.236723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761935.684621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761936.260748:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761936.708628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:3.0:1787761936.708829:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:3.0:1787761936.708876:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787761936.708896:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:3.0:1787761936.708935:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787761936.709020:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761933/real 1787761936] req@ffff9ac4523bda40 x1874603274187520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761936.709050:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274187520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761936.709064:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761936.709070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761936.709080:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187520:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787761936.709101:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761933/real 1787761936] req@ffff9ac4523bf840 x1874603274187392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761936.709117:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274187392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761936.709125:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761936.709146:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761936.709151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187392:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787761937.284663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761937.732654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761938.308651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761938.756669:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761938.756683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1787761938.756688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761938.756689:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761938.756711:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 983 00000100:00080000:0.0:1787761938.756739:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761938.756753:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761938.756832:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274187648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761938.756871:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761938.756876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761938.756879:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761938.756884:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 983 00000100:00080000:0.0:1787761938.756890:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761938.756897:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761938.756911:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761938.756941:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274187776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761938.756941:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187648:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761938.757063:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761938.757144:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761938/real 1787761938] req@ffff9ac451f85680 x1874603274187648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761938.757166:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274187648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761938.757178:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761938.757183:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761938.757189:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761938.757219:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187776:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761938.757248:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761938.757263:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761938/real 1787761938] req@ffff9ac451f843c0 x1874603274187776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761938.757281:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274187776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761938.757292:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761938.757297:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761938.757301:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274187776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761939.332737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761939.780587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761940.356706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761940.804660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761941.380655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761941.828593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761942.404749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761942.852707:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761943.428638:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761943.876619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761943.876619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761943.876628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761943.876633:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761943.876643:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 988 00000100:00080000:2.0:1787761943.876653:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761943.876660:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761943.876693:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274187904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761943.876716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761943.876719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761943.876721:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761943.876724:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 988 00000100:00080000:2.0:1787761943.876727:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761943.876730:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761943.876738:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761943.876750:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761943.876751:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274188032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761943.876809:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761943.876852:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761943/real 1787761943] req@ffff9ac4523bc3c0 x1874603274187904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761943.876867:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274187904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761943.876876:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761943.876879:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761943.876882:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274187904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761943.876901:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188032:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761943.876916:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761943.876925:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761943/real 1787761943] req@ffff9ac4523be940 x1874603274188032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761943.876938:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274188032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761943.876947:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761943.876951:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761943.876955:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761944.452726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761944.900644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761945.476773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761945.924626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761946.501602:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761946.948610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761947.524668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787761947.972603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761948.548750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761948.996621:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787761948.996621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761948.996630:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761948.996634:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761948.996641:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 993 00000100:00080000:0.0:1787761948.996651:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761948.996657:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761948.996705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274188160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761948.996727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761948.996730:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761948.996732:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761948.996737:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 993 00000100:00080000:0.0:1787761948.996742:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761948.996748:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761948.996759:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761948.996759:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761948.996779:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274188288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761948.996837:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761948.996879:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761948/real 1787761948] req@ffff9ac451f843c0 x1874603274188160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761948.996894:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274188160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761948.996902:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761948.996906:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761948.996909:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761948.996926:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188288:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761948.996943:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761948.996951:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761948/real 1787761948] req@ffff9ac451f85680 x1874603274188288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761948.996960:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274188288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761948.996966:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761948.996969:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761948.996971:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188288:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761949.572805:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761950.020599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761950.596633:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761951.044624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761951.620647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761952.068616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761952.644701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761953.092652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761953.668738:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761954.116630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761954.116648:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761954.116655:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761954.116659:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761954.116668:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 998 00000100:00080000:0.0:1787761954.116678:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761954.116685:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761954.116718:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274188416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761954.116739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761954.116743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761954.116744:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761954.116747:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 998 00000100:00080000:0.0:1787761954.116761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761954.116765:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761954.116773:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761954.116785:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787761954.116786:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274188544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761954.116846:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761954.116887:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761954/real 1787761954] req@ffff9ac451f861c0 x1874603274188416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761954.116902:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274188416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761954.116911:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761954.116914:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761954.116917:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761954.116936:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188544:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761954.116951:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761954.116959:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761954/real 1787761954] req@ffff9ac451f84780 x1874603274188544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761954.116969:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274188544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761954.116978:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761954.116982:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761954.116985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761954.692630:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761955.140589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761955.716633:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761956.164577:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761956.740661:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761957.188586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761957.764655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761958.212592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761958.788635:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761959.236585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761959.236612:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761959.236619:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761959.236624:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761959.236634:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1003 00000100:00080000:0.0:1787761959.236646:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761959.236653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761959.236692:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274188672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761959.236722:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761959.236726:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761959.236728:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761959.236733:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1003 00000100:00080000:0.0:1787761959.236736:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761959.236749:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761959.236759:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761959.236775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274188800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761959.236828:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188672:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761959.236952:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761959.237021:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761959/real 1787761959] req@ffff9ac451f84000 x1874603274188672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761959.237072:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274188672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761959.237090:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761959.237097:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761959.237102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761959.237113:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188800:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761959.237147:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761959.237163:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761959/real 1787761959] req@ffff9ac451f870c0 x1874603274188800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761959.237179:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274188800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761959.237190:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761959.237195:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761959.237199:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274188800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761959.812694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761960.260595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761960.836678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761961.284582:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761961.860712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761962.308579:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761962.884700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761963.332634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761963.908676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761964.356645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761964.356712:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761964.356720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761964.356724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761964.356735:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1009 00000100:00080000:2.0:1787761964.356747:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761964.356753:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761964.356794:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274188928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761964.356820:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761964.356823:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761964.356825:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761964.356828:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1009 00000100:00080000:2.0:1787761964.356831:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761964.356834:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761964.356862:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761964.356879:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274189056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761964.356935:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188928:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761964.357042:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761964.357109:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761964/real 1787761964] req@ffff9ac4523bc000 x1874603274188928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761964.357125:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274188928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761964.357148:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761964.357152:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761964.357155:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274188928:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761964.357177:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189056:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761964.357209:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761964.357220:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761964/real 1787761964] req@ffff9ac4523bd680 x1874603274189056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761964.357228:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274189056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761964.357234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761964.357237:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761964.357239:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189056:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761964.932802:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761965.380633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761965.956632:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761966.404587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761966.980624:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761967.428588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761968.004625:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761968.452588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761969.028706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761969.476598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761969.476619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761969.476625:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761969.476645:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761969.476653:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1014 00000100:00080000:2.0:1787761969.476663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761969.476668:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761969.476699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274189184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761969.476722:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761969.476725:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761969.476727:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761969.476730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1014 00000100:00080000:2.0:1787761969.476732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761969.476735:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761969.476742:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761969.476752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274189312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761969.476805:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189184:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761969.476861:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761969.476896:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761969/real 1787761969] req@ffff9ac4523bc780 x1874603274189184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761969.476909:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274189184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761969.476917:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761969.476920:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761969.476923:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761969.476936:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189312:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761969.476953:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761969.476962:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761969/real 1787761969] req@ffff9ac4523be580 x1874603274189312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761969.476970:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274189312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761969.476976:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761969.476978:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761969.476980:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761970.052667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761970.500601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761971.076686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761971.524585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761972.100749:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761972.548591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761973.124694:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761973.572608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761974.148645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787761974.596595:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761974.596603:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761974.596606:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787761974.596610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761974.596616:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1019 00000100:00080000:2.0:1787761974.596626:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761974.596640:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761974.596672:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274189440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761974.596694:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761974.596712:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761974.596714:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761974.596719:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1019 00000100:00080000:2.0:1787761974.596724:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761974.596729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761974.596740:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787761974.596752:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787761974.596754:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274189568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787761974.596816:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761974.596851:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761974/real 1787761974] req@ffff9ac4523bcf00 x1874603274189440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761974.596864:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274189440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787761999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761974.596871:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761974.596874:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761974.596878:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761974.596896:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189568:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761974.596912:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761974.596920:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761974/real 1787761974] req@ffff9ac4523bcb40 x1874603274189568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761974.596935:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274189568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787761999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761974.596941:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761974.596943:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761974.596945:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189568:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761975.172679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761975.620598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761976.196714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761976.644594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761977.220706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761977.668606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761978.244633:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787761978.692626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761979.268739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787761979.716646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761979.716662:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761979.716668:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787761979.716675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761979.716701:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1024 00000100:00080000:0.0:1787761979.716719:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761979.716732:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761979.716789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274189696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761979.716829:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761979.716834:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761979.716836:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761979.716842:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1024 00000100:00080000:0.0:1787761979.716846:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761979.716851:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761979.716887:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761979.716907:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274189824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761979.716918:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189696:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761979.717001:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761979.717041:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761979/real 1787761979] req@ffff9ac451f87c00 x1874603274189696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761979.717056:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274189696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761979.717066:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761979.717069:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761979.717072:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189696:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761979.717253:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189824:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761979.717275:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761979.717288:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761979/real 1787761979] req@ffff9ac451f84b40 x1874603274189824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761979.717301:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274189824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761979.717330:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761979.717334:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761979.717336:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274189824:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761980.292741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761980.740605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761981.316688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761981.764611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761982.340661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761982.788584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761983.364736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761983.812648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761984.388743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761984.836661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761984.836737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761984.836747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761984.836753:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761984.836767:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1029 00000100:00080000:0.0:1787761984.836780:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761984.836789:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761984.836837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274189952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761984.836872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761984.836877:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761984.836879:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761984.836885:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1029 00000100:00080000:0.0:1787761984.836889:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761984.836894:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761984.836918:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761984.836935:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274190080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761984.836951:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189952:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761984.837093:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761984.837144:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761984/real 1787761984] req@ffff9ac451f84b40 x1874603274189952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761984.837180:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274189952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761984.837201:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761984.837209:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761984.837218:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274189952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761984.837257:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190080:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761984.837292:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761984.837309:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761984/real 1787761984] req@ffff9ac451f87c00 x1874603274190080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761984.837325:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274190080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761984.837337:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761984.837341:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761984.837346:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190080:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761985.412694:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761985.860650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761986.436782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761986.884617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761987.460757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761987.908631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761988.484797:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761988.932671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761989.508786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761989.956666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787761989.956735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761989.956743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787761989.956749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761989.956762:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1034 00000100:00080000:2.0:1787761989.956775:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761989.956786:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787761989.956833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274190208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787761989.956904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787761989.956910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787761989.956913:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787761989.956918:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1034 00000100:00080000:2.0:1787761989.956924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787761989.956929:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787761989.956942:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787761989.956960:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274190336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761989.957066:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190208:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761989.957150:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761989.957200:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761989/real 1787761989] req@ffff9ac4523be1c0 x1874603274190208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761989.957244:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274190208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761989.957258:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761989.957263:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761989.957268:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761989.957278:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190336:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761989.957305:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761989.957319:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761989/real 1787761989] req@ffff9ac4523bf0c0 x1874603274190336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761989.957334:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274190336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761989.957344:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761989.957348:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761989.957352:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190336:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761990.532788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761990.980612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761991.556733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761992.004650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761992.580673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761993.028594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761993.604754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761994.052671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761994.628780:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761995.076663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787761995.076728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761995.076738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787761995.076743:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761995.076754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1039 00000100:00080000:0.0:1787761995.076795:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761995.076802:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787761995.076846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274190464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787761995.076883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787761995.076886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787761995.076888:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787761995.076891:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1039 00000100:00080000:0.0:1787761995.076894:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787761995.076897:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787761995.076909:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787761995.076918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274190592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787761995.076932:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190464:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787761995.077023:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761995.077064:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761995/real 1787761995] req@ffff9ac451f870c0 x1874603274190464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761995.077087:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274190464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761995.077097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761995.077101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761995.077104:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787761995.077130:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190592:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787761995.077148:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787761995.077157:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787761995/real 1787761995] req@ffff9ac451f84000 x1874603274190592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787761995.077165:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274190592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787761995.077172:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787761995.077174:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787761995.077176:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190592:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787761995.652733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761996.100614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761996.676730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761997.124655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761997.700757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761998.148638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761998.724814:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787761999.172622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787761999.748853:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762000.196634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762000.196634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762000.196652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762000.196658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762000.196671:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1044 00000100:00080000:0.0:1787762000.196686:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762000.196693:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762000.196738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274190720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762000.196767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762000.196771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762000.196789:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762000.196794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1044 00000100:00080000:0.0:1787762000.196797:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762000.196800:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762000.196812:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762000.196824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274190848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762000.196865:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762000.197027:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190848:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762000.772763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762001.220742:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762001.796759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762002.244712:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762002.820773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762003.268818:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762003.269037:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762003.269095:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762003.269116:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762003.269174:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762003.269285:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762000/real 1787762003] req@ffff9ac451f84780 x1874603274190720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762003.269334:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274190720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762003.269357:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762003.269367:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762003.269375:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762003.269392:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762000/real 1787762003] req@ffff9ac451f861c0 x1874603274190848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762003.269419:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274190848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762003.269437:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762003.269444:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762003.269451:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274190848:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762003.844784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762004.292729:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762004.868783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762005.316664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762005.316692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762005.316697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762005.316699:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762005.316720:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1050 00000100:00080000:0.0:1787762005.316734:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762005.316745:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762005.316790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274190976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762005.316857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762005.316865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762005.316869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762005.316877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1050 00000100:00080000:0.0:1787762005.316885:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762005.316893:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762005.316909:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762005.316968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274191104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762005.317011:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190976:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762005.317118:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762005.317186:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762005/real 1787762005] req@ffff9ac451f861c0 x1874603274190976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762005.317221:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274190976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762005.317241:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762005.317250:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762005.317258:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274190976:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762005.317299:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191104:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762005.317348:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762005.317376:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762005/real 1787762005] req@ffff9ac451f84780 x1874603274191104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762005.317401:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274191104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762005.317419:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762005.317426:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762005.317434:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191104:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762005.892764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762006.340800:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762006.916764:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762007.364679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762007.941359:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762008.388658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762008.964873:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762009.412606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762009.988652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762010.436591:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762010.436591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762010.436601:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762010.436605:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762010.436614:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1055 00000100:00080000:0.0:1787762010.436623:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762010.436630:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762010.436671:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274191232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762010.436711:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762010.436716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762010.436718:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762010.436724:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1055 00000100:00080000:0.0:1787762010.436728:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762010.436734:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762010.436745:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762010.436762:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274191360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762010.436765:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274191232:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762010.436825:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762010.436858:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762010/real 1787762010] req@ffff9ac451f85680 x1874603274191232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762010.436898:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274191232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762010.436906:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762010.436909:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762010.436912:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274191232:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762010.436935:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191360:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762010.436954:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762010.436965:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762010/real 1787762010] req@ffff9ac451f843c0 x1874603274191360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762010.436974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274191360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762010.436981:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762010.436983:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762010.436985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191360:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762011.012814:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762011.460664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762012.036736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762012.484669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762013.060957:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762013.508613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762014.084663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762014.532625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762015.108729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762015.556607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762015.556619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762015.556627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762015.556631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762015.556639:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1060 00000100:00080000:0.0:1787762015.556650:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762015.556657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762015.556693:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274191488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762015.556720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762015.556723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762015.556725:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762015.556729:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1060 00000100:00080000:0.0:1787762015.556732:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762015.556735:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762015.556755:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762015.556767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274191616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762015.556814:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274191488:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762015.556897:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762015.556946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762015/real 1787762015] req@ffff9ac451f843c0 x1874603274191488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762015.556962:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274191488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762015.556986:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762015.556990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762015.556994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274191488:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762015.557001:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191616:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762015.557024:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762015.557035:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762015/real 1787762015] req@ffff9ac451f85680 x1874603274191616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762015.557046:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274191616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762015.557053:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762015.557056:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762015.557059:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191616:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762016.132746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762016.580629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762017.156783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762017.604617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762018.180773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762018.628657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762019.204736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762019.652627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762020.228732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762020.676650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762020.676713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762020.676727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762020.676735:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762020.676751:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1065 00000100:00080000:1.0:1787762020.676770:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762020.676779:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762020.676855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274191744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762020.676905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762020.676930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762020.676934:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762020.676942:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1065 00000100:00080000:1.0:1787762020.676950:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762020.676958:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762020.676972:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762020.676976:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274191744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762020.676996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274191872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762020.677083:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762020.677137:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762020/real 1787762020] req@ffff9ac447840780 x1874603274191744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762020.677161:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274191744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762020.677175:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762020.677181:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762020.677186:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274191744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762020.677214:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191872:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762020.677243:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762020.677258:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762020/real 1787762020] req@ffff9ac447840f00 x1874603274191872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762020.677274:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274191872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762020.677285:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762020.677289:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762020.677293:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274191872:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762021.252783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762021.700644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762022.276754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762022.724606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762023.300815:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762023.748597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762024.324711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762024.772644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762025.348802:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762025.796629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762025.796717:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762025.796727:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762025.796733:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762025.796745:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1070 00000100:00080000:1.0:1787762025.796757:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762025.796767:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762025.796838:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274192000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762025.796871:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762025.796876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762025.796880:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762025.796886:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1070 00000100:00080000:1.0:1787762025.796892:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762025.796898:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762025.796907:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192000:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762025.796915:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762025.796934:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274192128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762025.797017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762025.797067:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762025/real 1787762025] req@ffff9ac447840000 x1874603274192000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762025.797091:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274192000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762025.797105:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762025.797110:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762025.797115:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192000:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762025.797143:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192128:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762025.797168:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762025.797183:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762025/real 1787762025] req@ffff9ac447842580 x1874603274192128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762025.797198:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274192128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762025.797209:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762025.797213:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762025.797217:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192128:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762026.372733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762026.820638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762027.396736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762027.844651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762028.420744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762028.868656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762029.444719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762029.892644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762030.468800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762030.916616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762030.916650:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762030.916656:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762030.916660:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762030.916669:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1075 00000100:00080000:0.0:1787762030.916678:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762030.916700:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762030.916730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274192256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762030.916770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762030.916772:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762030.916774:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762030.916777:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1075 00000100:00080000:0.0:1787762030.916781:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762030.916785:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762030.916794:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762030.916801:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192256:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762030.916807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274192384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762030.916860:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762030.916893:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762030/real 1787762030] req@ffff9ac451f84780 x1874603274192256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762030.916906:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274192256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762030.916914:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762030.916917:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762030.916919:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192256:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762030.916935:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192384:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762030.916950:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762030.916958:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762030/real 1787762030] req@ffff9ac451f861c0 x1874603274192384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762030.916966:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274192384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762030.916971:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762030.916973:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762030.916975:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192384:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762031.492727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762031.940653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762032.516763:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762032.964599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762033.540666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762033.988667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762034.564804:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762035.012611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762035.588728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762036.036608:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762036.036613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762036.036622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762036.036628:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762036.036639:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1080 00000100:00080000:0.0:1787762036.036652:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762036.036660:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762036.036719:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274192512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762036.036747:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762036.036750:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762036.036752:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762036.036756:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1080 00000100:00080000:0.0:1787762036.036780:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762036.036784:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762036.036793:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192512:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787762036.036796:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762036.036810:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274192640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762036.036878:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762036.036932:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762036/real 1787762036] req@ffff9ac451f84000 x1874603274192512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762036.036946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274192512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762036.036955:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762036.036958:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762036.036961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192512:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762036.036984:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192640:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762036.037001:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762036.037010:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762036/real 1787762036] req@ffff9ac451f870c0 x1874603274192640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762036.037018:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274192640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762036.037024:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762036.037027:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762036.037029:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192640:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762036.612770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762037.060635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762037.636717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762038.084641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762038.660696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762039.108600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762039.684644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762040.132644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762040.708684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762041.156598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762041.156606:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762041.156612:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762041.156627:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762041.156638:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1085 00000100:00080000:0.0:1787762041.156652:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762041.156660:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762041.156719:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274192768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762041.156746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762041.156749:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762041.156751:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762041.156756:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1085 00000100:00080000:0.0:1787762041.156760:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762041.156764:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762041.156778:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762041.156794:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274192896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762041.156827:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192768:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762041.156925:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762041.156997:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762041/real 1787762041] req@ffff9ac451f87c00 x1874603274192768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762041.157022:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274192768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762041.157065:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762041.157070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762041.157076:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274192768:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762041.157108:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192896:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762041.157142:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762041.157158:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762041/real 1787762041] req@ffff9ac451f84b40 x1874603274192896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762041.157174:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274192896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762041.157185:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762041.157189:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762041.157194:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274192896:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762041.732683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762042.180577:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762042.756669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762043.204598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762043.780635:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762044.228621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762044.804690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762045.252614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762045.828705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762046.276682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762046.276738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762046.276756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762046.276763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762046.276780:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1091 00000100:00080000:1.0:1787762046.276797:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762046.276810:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762046.276868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274193024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762046.276928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762046.276963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762046.276966:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762046.276973:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1091 00000100:00080000:1.0:1787762046.276979:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762046.276985:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762046.276998:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762046.277016:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274193152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762046.277110:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193024:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762046.277267:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762046.277322:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762046/real 1787762046] req@ffff9ac447843840 x1874603274193024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762046.277343:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274193024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762046.277357:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762046.277361:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762046.277366:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193024:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762046.277375:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193152:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762046.277397:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762046.277408:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762046/real 1787762046] req@ffff9ac447841e00 x1874603274193152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762046.277420:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274193152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762046.277429:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762046.277433:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762046.277436:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193152:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762046.852762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762047.300652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762047.876767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762048.324702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762048.900810:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762049.348687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762049.924769:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762050.372653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762050.948774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762051.396687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762051.396737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762051.396747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762051.396753:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762051.396765:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1096 00000100:00080000:1.0:1787762051.396779:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762051.396788:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762051.396839:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274193280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762051.396888:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762051.396896:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762051.396900:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762051.396908:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1096 00000100:00080000:1.0:1787762051.396917:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762051.396927:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762051.396947:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762051.396975:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274193408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762051.396993:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193280:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762051.397147:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762051.397220:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762051/real 1787762051] req@ffff9ac447841a40 x1874603274193280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762051.397274:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274193280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762051.397293:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762051.397302:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762051.397309:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193280:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762051.397350:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193408:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762051.397383:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762051.397399:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762051/real 1787762051] req@ffff9ac4478412c0 x1874603274193408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762051.397413:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274193408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762051.397423:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762051.397427:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762051.397430:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193408:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762051.972775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762052.420617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762052.996663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762053.444677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762054.020767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762054.468679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762055.044737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762055.492680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762056.068747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762056.516639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762056.516721:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762056.516734:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762056.516741:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762056.516757:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1101 00000100:00080000:0.0:1787762056.516773:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762056.516787:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762056.516845:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274193536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762056.516909:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762056.516916:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762056.516920:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762056.516929:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1101 00000100:00080000:0.0:1787762056.516937:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762056.516945:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762056.516964:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762056.516992:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274193664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762056.517005:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193536:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762056.517116:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762056.517167:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762056/real 1787762056] req@ffff9ac451f84b40 x1874603274193536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762056.517183:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274193536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762056.517193:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762056.517197:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762056.517200:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193536:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762056.517226:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193664:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762056.517242:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762056.517251:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762056/real 1787762056] req@ffff9ac451f87c00 x1874603274193664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762056.517259:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274193664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762056.517265:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762056.517267:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762056.517270:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193664:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762057.092648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762057.540603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762058.116741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762058.564705:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762059.140645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762059.588663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762060.164673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762060.612614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762061.188648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762061.636604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762061.636620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762061.636627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762061.636652:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762061.636663:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1106 00000100:00080000:1.0:1787762061.636677:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762061.636684:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762061.636726:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274193792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762061.636755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762061.636758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762061.636760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762061.636763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1106 00000100:00080000:1.0:1787762061.636766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762061.636769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762061.636780:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762061.636795:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762061.636795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274193920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762061.636878:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762061.636921:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762061/real 1787762061] req@ffff9ac447843480 x1874603274193792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762061.636936:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274193792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762061.636946:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762061.636949:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762061.636953:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274193792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762061.636976:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193920:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762061.636992:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762061.637000:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762061/real 1787762061] req@ffff9ac4478412c0 x1874603274193920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762061.637017:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274193920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762061.637022:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762061.637025:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762061.637027:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274193920:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762062.212800:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762062.660654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762063.236725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762063.684635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762064.260755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762064.708659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762065.284782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762065.732656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762066.308761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762066.756639:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762066.756653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762066.756660:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762066.756661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762066.756675:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1111 00000100:00080000:1.0:1787762066.756690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762066.756701:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762066.756749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274194048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762066.756785:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762066.756805:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762066.756808:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762066.756815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1111 00000100:00080000:1.0:1787762066.756823:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762066.756830:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762066.756843:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762066.756856:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194048:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762066.756868:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274194176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762066.757023:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194176:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762067.332753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762067.780652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762068.356732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762068.804646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762069.380775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762069.828620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762069.828953:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762069.829017:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762069.829041:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762069.829107:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762069.829184:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762066/real 1787762069] req@ffff9ac447841e00 x1874603274194176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762069.829223:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274194176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762069.829245:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762069.829255:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762069.829262:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194176:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787762069.829291:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762066/real 1787762069] req@ffff9ac447841a40 x1874603274194048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762069.829311:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274194048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762069.829322:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762069.829327:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762069.829365:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194048:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787762070.404732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762070.852656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762071.428697:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762071.876638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762071.876780:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762071.876789:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762071.876795:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762071.876806:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1116 00000100:00080000:1.0:1787762071.876819:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762071.876828:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762071.876872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274194304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762071.876906:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762071.876912:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762071.876914:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762071.876919:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1116 00000100:00080000:1.0:1787762071.876939:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762071.876945:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762071.876957:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762071.876975:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274194432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762071.877020:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194304:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762071.877124:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762071.877200:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762071/real 1787762071] req@ffff9ac447843840 x1874603274194304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762071.877233:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274194304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762071.877253:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762071.877270:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762071.877290:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194304:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762071.877304:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194432:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762071.877352:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762071.877377:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762071/real 1787762071] req@ffff9ac447842580 x1874603274194432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762071.877405:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274194432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762071.877422:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762071.877429:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762071.877437:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194432:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762072.452707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762072.900634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762073.476720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762073.924650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762074.500775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762074.948657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762075.524749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762075.972593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762076.548707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762076.996633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762076.996684:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762076.996695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762076.996701:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762076.996713:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1121 00000100:00080000:1.0:1787762076.996726:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762076.996736:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762076.996785:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274194560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762076.996819:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762076.996827:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762076.996832:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762076.996842:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1121 00000100:00080000:1.0:1787762076.996849:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762076.996866:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762076.996868:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194560:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762076.996894:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762076.996934:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274194688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762076.996977:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762076.997052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762076/real 1787762076] req@ffff9ac447840000 x1874603274194560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762076.997070:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274194560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762076.997080:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762076.997084:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762076.997087:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194560:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762076.997114:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194688:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762076.997132:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762076.997141:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762076/real 1787762076] req@ffff9ac447840f00 x1874603274194688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762076.997150:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274194688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762076.997156:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762076.997158:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762076.997160:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194688:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762077.573313:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762078.020652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762078.596771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762079.044615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762079.620935:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762080.068657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762080.644779:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762081.092653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762081.668774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762082.116670:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762082.116675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762082.116683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762082.116696:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762082.116710:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1126 00000100:00080000:0.0:1787762082.116725:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762082.116734:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762082.116782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274194816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762082.116833:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762082.116837:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762082.116839:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762082.116843:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1126 00000100:00080000:0.0:1787762082.116846:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762082.116849:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762082.116861:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762082.116876:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274194944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762082.116905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194816:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762082.117050:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762082.117133:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762082/real 1787762082] req@ffff9ac451f870c0 x1874603274194816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762082.117168:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274194816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762082.117184:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762082.117191:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762082.117196:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274194816:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762082.117230:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194944:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762082.117266:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762082.117284:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762082/real 1787762082] req@ffff9ac451f84000 x1874603274194944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762082.117301:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274194944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762082.117312:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762082.117317:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762082.117321:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274194944:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762082.692730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762083.140652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762083.716655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762084.164612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762084.740753:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762085.188639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762085.764779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762086.212629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762086.788763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762087.236671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762087.236735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762087.236742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762087.236747:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762087.236760:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1131 00000100:00080000:2.0:1787762087.236775:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762087.236786:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762087.236837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274195072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762087.236884:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762087.236890:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762087.236893:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762087.236899:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1131 00000100:00080000:2.0:1787762087.236904:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762087.236910:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762087.236924:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762087.236941:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274195200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762087.236954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195072:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762087.237080:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762087.237134:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762087/real 1787762087] req@ffff9ac4523bf0c0 x1874603274195072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762087.237155:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274195072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762087.237168:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762087.237173:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762087.237178:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195072:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762087.237211:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195200:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762087.237233:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762087.237243:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762087/real 1787762087] req@ffff9ac4523be1c0 x1874603274195200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762087.237273:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274195200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762087.237281:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762087.237285:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762087.237288:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195200:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762087.812752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762088.260615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762088.836729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762089.284629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762089.860739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762090.308688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762090.884650:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762091.332610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762091.908665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762092.356613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762092.356630:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762092.356637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762092.356642:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762092.356653:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1137 00000100:00080000:1.0:1787762092.356667:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762092.356701:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762092.356743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274195328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762092.356768:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762092.356771:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762092.356773:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762092.356776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1137 00000100:00080000:1.0:1787762092.356779:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762092.356782:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762092.356793:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762092.356802:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274195456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762092.356810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195328:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762092.356892:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762092.356942:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762092/real 1787762092] req@ffff9ac447840780 x1874603274195328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762092.356963:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274195328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762092.356977:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762092.356980:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762092.356984:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195328:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762092.357009:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195456:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762092.357030:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762092.357040:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762092/real 1787762092] req@ffff9ac447841680 x1874603274195456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762092.357049:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274195456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762092.357072:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762092.357076:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762092.357080:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195456:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762092.932688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762093.380598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762093.956689:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762094.404602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762094.980696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762095.428596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762096.004735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762096.452595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762097.028681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762097.476658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762097.476712:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762097.476727:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762097.476732:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762097.476746:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1142 00000100:00080000:1.0:1787762097.476760:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762097.476770:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762097.476825:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274195584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762097.476876:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762097.476884:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762097.476888:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762097.476898:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1142 00000100:00080000:1.0:1787762097.476929:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762097.476937:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762097.476952:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762097.476983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762097.476983:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274195712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762097.477095:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762097.477149:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762097/real 1787762097] req@ffff9ac4478403c0 x1874603274195584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762097.477173:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274195584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762097.477187:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762097.477192:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762097.477198:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762097.477239:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195712:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762097.477267:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762097.477311:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762097/real 1787762097] req@ffff9ac4478421c0 x1874603274195712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762097.477327:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274195712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762097.477338:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762097.477343:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762097.477347:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195712:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762098.052733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762098.500593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762099.076670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762099.524617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762100.100750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762100.548638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762101.124721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762101.572686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762102.148750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762102.596614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762102.596717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762102.596729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762102.596735:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762102.596747:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1147 00000100:00080000:0.0:1787762102.596761:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762102.596770:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762102.596834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274195840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762102.596875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762102.596880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762102.596883:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762102.596890:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1147 00000100:00080000:0.0:1787762102.596895:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762102.596901:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762102.596914:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762102.596926:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195840:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762102.596933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274195968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762102.597041:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762102.597093:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762102/real 1787762102] req@ffff9ac451f861c0 x1874603274195840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762102.597117:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274195840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762102.597128:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762102.597131:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762102.597135:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274195840:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762102.597163:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195968:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762102.597182:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762102.597192:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762102/real 1787762102] req@ffff9ac451f84780 x1874603274195968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762102.597200:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274195968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762102.597207:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762102.597209:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762102.597232:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274195968:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762103.172741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762103.620634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762104.196705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762104.644640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762105.220664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762105.668600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762106.244673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762106.692635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762107.268722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762107.716652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762107.716653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762107.716668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762107.716674:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762107.716687:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1152 00000100:00080000:0.0:1787762107.716701:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762107.716711:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762107.716758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274196096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762107.716818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762107.716824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762107.716826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762107.716832:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1152 00000100:00080000:0.0:1787762107.716837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762107.716843:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762107.716855:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762107.716872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274196224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762107.716986:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196096:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762107.717069:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762107.717133:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762107/real 1787762107] req@ffff9ac451f85680 x1874603274196096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762107.717163:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274196096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762107.717177:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762107.717183:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762107.717189:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196096:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762107.717198:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196224:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762107.717260:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762107.717277:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762107/real 1787762107] req@ffff9ac451f843c0 x1874603274196224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762107.717293:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274196224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762107.717304:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762107.717308:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762107.717313:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196224:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762108.292712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762108.740630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762109.316733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762109.764622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762110.340731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762110.788621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762111.364735:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762111.812656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762112.388735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762112.836593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762112.836655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762112.836667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762112.836672:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762112.836697:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1157 00000100:00080000:0.0:1787762112.836711:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762112.836720:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762112.836779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274196352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762112.836832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762112.836838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762112.836841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762112.836849:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1157 00000100:00080000:0.0:1787762112.836856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762112.836862:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762112.836880:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196352:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787762112.836913:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762112.836950:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274196480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762112.836989:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762112.837067:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762112/real 1787762112] req@ffff9ac451f843c0 x1874603274196352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762112.837085:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274196352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762112.837097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762112.837103:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762112.837111:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196352:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762112.837139:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196480:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762112.837158:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762112.837168:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762112/real 1787762112] req@ffff9ac451f85680 x1874603274196480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762112.837195:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274196480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762112.837205:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762112.837209:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762112.837213:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196480:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762113.412712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762113.860600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762114.436680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762114.884639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762115.460729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762115.908629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762116.484805:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762116.932669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762117.508836:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762117.956643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762117.956664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762117.956674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762117.956680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762117.956693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1162 00000100:00080000:1.0:1787762117.956707:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762117.956716:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762117.956761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274196608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762117.956829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762117.956834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762117.956837:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762117.956843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1162 00000100:00080000:1.0:1787762117.956849:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762117.956857:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762117.956872:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762117.956887:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762117.956895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274196736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762117.956981:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762117.957044:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762117/real 1787762117] req@ffff9ac4478430c0 x1874603274196608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762117.957067:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274196608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762117.957081:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762117.957087:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762117.957093:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762117.957121:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196736:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762117.957182:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762117.957201:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762117/real 1787762117] req@ffff9ac447842d00 x1874603274196736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762117.957218:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274196736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762117.957230:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762117.957234:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762117.957241:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196736:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762118.532737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762118.980622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762119.556805:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762120.004650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762120.580738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762121.028636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762121.604651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762122.052664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762122.628670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762123.076664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762123.076714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762123.076724:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762123.076730:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762123.076744:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1167 00000100:00080000:2.0:1787762123.076757:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762123.076781:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762123.076814:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274196864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762123.076837:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762123.076840:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762123.076841:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762123.076844:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1167 00000100:00080000:2.0:1787762123.076848:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762123.076853:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762123.076880:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762123.076902:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762123.076912:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274196992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762123.077014:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762123.077066:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762123/real 1787762123] req@ffff9ac4523bcb40 x1874603274196864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762123.077089:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274196864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762123.077148:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762123.077151:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762123.077154:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274196864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762123.077172:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196992:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762123.077189:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762123.077197:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762123/real 1787762123] req@ffff9ac4523bcf00 x1874603274196992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762123.077205:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274196992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762123.077211:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762123.077213:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762123.077215:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274196992:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762123.652642:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762124.100661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762124.676665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762125.124608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762125.700739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762126.148615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762126.724744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762127.172617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762127.748681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762128.196706:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762128.196752:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762128.196762:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762128.196768:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762128.196782:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1172 00000100:00080000:1.0:1787762128.196796:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762128.196807:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762128.196863:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274197120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762128.196916:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762128.196924:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762128.196927:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762128.196934:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1172 00000100:00080000:1.0:1787762128.196940:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762128.196957:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762128.196983:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762128.197008:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274197248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762128.197041:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197120:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762128.197185:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762128.197239:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762128/real 1787762128] req@ffff9ac447842940 x1874603274197120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762128.197321:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274197120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762128.197332:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762128.197337:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762128.197340:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762128.197385:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274197248:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762128.197411:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762128.197423:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762128/real 1787762128] req@ffff9ac447843c00 x1874603274197248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762128.197434:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274197248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762128.197442:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762128.197445:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762128.197448:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274197248:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762128.772727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762129.220661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762129.796799:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762130.244634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762130.820800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762131.268634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762131.844737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762132.292660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762132.868743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762133.316635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762133.316690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762133.316700:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762133.316706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762133.316717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1178 00000100:00080000:1.0:1787762133.316730:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762133.316740:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762133.316796:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274197376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762133.316829:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762133.316834:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762133.316836:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762133.316842:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1178 00000100:00080000:1.0:1787762133.316846:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762133.316851:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762133.316864:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762133.316883:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274197504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762133.316908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197376:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762133.317115:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274197504:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762133.892781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762134.340598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762134.916645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762135.364635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762135.940652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762136.388626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762136.388899:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762136.388963:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762136.388984:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762136.389074:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762136.389159:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762133/real 1787762136] req@ffff9ac447843c00 x1874603274197504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762136.389199:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274197504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762136.389251:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762136.389261:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762136.389269:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274197504:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787762136.389285:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762133/real 1787762136] req@ffff9ac447840b40 x1874603274197376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762136.389312:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274197376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762136.389329:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762136.389336:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762136.389343:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197376:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787762136.964731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762137.412626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762137.988699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762138.436614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762138.436765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762138.436783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762138.436790:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762138.436818:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1183 00000100:00080000:0.0:1787762138.436834:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762138.436847:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762138.436903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274197632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762138.436952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762138.436959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762138.436963:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762138.436971:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1183 00000100:00080000:0.0:1787762138.437033:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787762138.437046:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197632:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787762138.437049:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762138.437074:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762138.437110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274197760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762138.437141:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762138.437196:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762138/real 1787762138] req@ffff9ac451f84780 x1874603274197632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762138.437220:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274197632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762138.437234:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762138.437239:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762138.437247:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197632:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762138.437297:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274197760:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762138.437343:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762138.437369:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762138/real 1787762138] req@ffff9ac451f861c0 x1874603274197760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762138.437393:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274197760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762138.437406:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762138.437413:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762138.437420:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274197760:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762139.012744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762139.460651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762140.036661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762140.484646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762141.061066:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762141.508610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762142.084788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762142.532703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762143.108796:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762143.556672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762143.556693:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762143.556702:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762143.556706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762143.556730:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1188 00000100:00080000:1.0:1787762143.556740:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762143.556746:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762143.556778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274197888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762143.556803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762143.556806:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762143.556808:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762143.556811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1188 00000100:00080000:1.0:1787762143.556814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762143.556818:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762143.556827:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762143.556839:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762143.556840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274198016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762143.556899:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762143.556933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762143/real 1787762143] req@ffff9ac447842940 x1874603274197888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762143.556946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274197888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762143.556978:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762143.556981:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762143.556984:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274197888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762143.557006:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198016:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762143.557029:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762143.557042:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762143/real 1787762143] req@ffff9ac447842d00 x1874603274198016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762143.557066:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274198016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762143.557073:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762143.557075:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762143.557085:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198016:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762144.132812:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762144.580628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762145.156710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762145.604608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762146.180731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762146.628671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762147.204741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762147.652701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762148.228728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762148.676657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762148.676697:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762148.676714:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762148.676722:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762148.676738:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1193 00000100:00080000:1.0:1787762148.676756:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762148.676771:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762148.676825:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274198144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762148.676867:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762148.676873:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762148.676899:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762148.676911:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1193 00000100:00080000:1.0:1787762148.676922:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762148.676955:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762148.676967:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198144:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762148.676988:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762148.677026:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274198272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762148.677071:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762148.677125:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762148/real 1787762148] req@ffff9ac4478430c0 x1874603274198144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762148.677149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274198144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762148.677164:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762148.677169:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762148.677174:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198144:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762148.677204:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198272:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762148.677231:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762148.677245:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762148/real 1787762148] req@ffff9ac4478421c0 x1874603274198272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762148.677261:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274198272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762148.677272:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762148.677276:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762148.677280:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198272:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762149.252749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762149.700670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762150.276653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762150.724653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762151.300645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762151.748599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762152.324648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762152.772613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762153.348651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762153.796591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762153.796619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762153.796625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762153.796629:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762153.796638:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1198 00000100:00080000:1.0:1787762153.796647:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762153.796653:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762153.796684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274198400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762153.796706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762153.796708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762153.796710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762153.796712:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1198 00000100:00080000:1.0:1787762153.796715:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762153.796718:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762153.796726:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762153.796736:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274198528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762153.796752:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198400:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762153.796818:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762153.796859:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762153/real 1787762153] req@ffff9ac4478403c0 x1874603274198400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762153.796872:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274198400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762153.796880:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762153.796883:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762153.796886:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198400:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762153.796918:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198528:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762153.796939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762153.796952:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762153/real 1787762153] req@ffff9ac447841680 x1874603274198528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762153.796966:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274198528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762153.796976:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762153.796980:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762153.796983:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198528:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762154.372737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762154.820683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762155.396767:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762155.844667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762156.420715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762156.868618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762157.444787:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762157.892687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762158.468770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762158.916694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762158.916711:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762158.916722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762158.916729:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762158.916742:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1203 00000100:00080000:2.0:1787762158.916776:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762158.916788:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762158.916834:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274198656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762158.916898:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762158.916905:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762158.916908:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762158.916915:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1203 00000100:00080000:2.0:1787762158.916921:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762158.916929:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762158.916944:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762158.916960:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198656:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762158.916969:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274198784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762158.917053:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762158.917105:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762158/real 1787762158] req@ffff9ac4523be580 x1874603274198656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762158.917150:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274198656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762158.917165:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762158.917170:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762158.917176:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198656:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762158.917207:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198784:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762158.917238:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762158.917253:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762158/real 1787762158] req@ffff9ac4523bc780 x1874603274198784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762158.917268:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274198784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762158.917279:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762158.917283:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762158.917288:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274198784:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762159.492832:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762159.940666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762160.516754:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762160.964685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762161.540727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762161.988600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762162.564661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762163.012644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762163.588746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762164.036668:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762164.036670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762164.036685:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762164.036692:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762164.036705:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1208 00000100:00080000:1.0:1787762164.036718:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762164.036728:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762164.036776:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274198912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762164.036805:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762164.036811:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762164.036814:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762164.036851:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1208 00000100:00080000:1.0:1787762164.036860:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762164.036867:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762164.036876:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198912:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762164.036887:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762164.036914:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274199040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762164.036970:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762164.037040:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762164/real 1787762164] req@ffff9ac447840780 x1874603274198912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762164.037085:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274198912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762164.037100:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762164.037106:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762164.037111:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274198912:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762164.037142:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199040:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762164.037175:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762164.037192:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762164/real 1787762164] req@ffff9ac447840f00 x1874603274199040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762164.037207:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274199040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762164.037218:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762164.037223:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762164.037227:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199040:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762164.612734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762165.060635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762165.636746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762166.084664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762166.660649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762167.108603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762167.684698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762168.132621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762168.708668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762169.156598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762169.156620:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762169.156626:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762169.156630:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762169.156638:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1213 00000100:00080000:2.0:1787762169.156647:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762169.156653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762169.156684:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274199168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762169.156706:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762169.156709:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762169.156724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762169.156728:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1213 00000100:00080000:2.0:1787762169.156732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787762169.156736:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199168:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787762169.156739:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762169.156750:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762169.156763:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274199296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762169.156811:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762169.156846:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762169/real 1787762169] req@ffff9ac4523bd680 x1874603274199168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762169.156868:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274199168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762169.156876:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762169.156880:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762169.156883:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199168:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762169.156903:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199296:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762169.156920:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762169.156928:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762169/real 1787762169] req@ffff9ac4523bc000 x1874603274199296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762169.156936:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274199296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762169.156942:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762169.156945:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762169.156947:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199296:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762169.732644:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762170.180578:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762170.756675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762171.204590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762171.780651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762172.228612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762172.804657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762173.252606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762173.828700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762174.276604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762174.276604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762174.276614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762174.276619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762174.276627:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1219 00000100:00080000:2.0:1787762174.276637:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762174.276644:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762174.276685:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274199424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762174.276705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762174.276708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762174.276710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762174.276713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1219 00000100:00080000:2.0:1787762174.276716:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762174.276719:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762174.276728:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762174.276736:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762174.276739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274199552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762174.276812:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762174.276847:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762174/real 1787762174] req@ffff9ac4523be940 x1874603274199424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762174.276868:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274199424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762174.276876:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762174.276879:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762174.276882:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762174.276903:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199552:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762174.276919:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762174.276928:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762174/real 1787762174] req@ffff9ac4523bc3c0 x1874603274199552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762174.276936:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274199552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762174.276942:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762174.276945:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762174.276947:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199552:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762174.852683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762175.300640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762175.876666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762176.324596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762176.900687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762177.348592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762177.924647:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762178.372612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762178.948704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787762179.396636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762179.396641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762179.396653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762179.396660:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762179.396675:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1224 00000100:00080000:2.0:1787762179.396689:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762179.396699:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762179.396769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274199680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762179.396803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762179.396808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762179.396811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762179.396816:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1224 00000100:00080000:2.0:1787762179.396821:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762179.396826:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762179.396840:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762179.396857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274199808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762179.396869:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199680:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762179.396958:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762179.397009:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762179/real 1787762179] req@ffff9ac4523bf840 x1874603274199680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762179.397052:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274199680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762179.397067:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762179.397072:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762179.397078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199680:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762179.397110:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199808:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762179.397140:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762179.397155:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762179/real 1787762179] req@ffff9ac4523bda40 x1874603274199808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762179.397188:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274199808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762179.397200:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762179.397204:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762179.397208:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274199808:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762179.972786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762180.420676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762180.996818:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762181.444615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762182.020647:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762182.468629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762183.044700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762183.492590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762184.068681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787762184.516613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762184.516622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762184.516626:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762184.516636:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1229 00000100:00100000:0.0:1787762184.516640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762184.516645:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762184.516653:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762184.516715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274199936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762184.516735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762184.516738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762184.516739:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762184.516742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1229 00000100:00080000:2.0:1787762184.516746:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762184.516749:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762184.516759:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762184.516767:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199936:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762184.516772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274200064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762184.516824:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762184.516858:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762184/real 1787762184] req@ffff9ac4523bda40 x1874603274199936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762184.516878:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274199936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762184.516887:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762184.516890:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762184.516893:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274199936:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762184.516912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200064:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762184.516929:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762184.516938:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762184/real 1787762184] req@ffff9ac4523bf840 x1874603274200064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762184.516946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274200064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762184.516952:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762184.516955:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762184.516964:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200064:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762185.092673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762185.540657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762186.116748:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762186.564640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762187.140737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762187.588685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762188.164728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762188.612699:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762189.188809:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762189.636739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762189.636755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762189.636758:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762189.636763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762189.636782:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1234 00000100:00080000:0.0:1787762189.636824:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762189.636838:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762189.636890:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274200192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762189.636943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762189.636959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762189.636964:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762189.636972:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1234 00000100:00080000:0.0:1787762189.636980:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762189.636988:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762189.637004:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762189.637031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274200320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762189.637059:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200192:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762189.637156:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762189.637217:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762189/real 1787762189] req@ffff9ac451f84000 x1874603274200192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762189.637240:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274200192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762189.637267:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762189.637289:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762189.637295:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200192:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762189.637326:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200320:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762189.637359:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762189.637376:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762189/real 1787762189] req@ffff9ac451f870c0 x1874603274200320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762189.637391:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274200320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762189.637402:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762189.637406:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762189.637421:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200320:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762190.212762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762190.660683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762191.236770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762191.684627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762192.260800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762192.708668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762193.284752:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762193.732657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762194.308794:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762194.756648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762194.756691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762194.756702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762194.756708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762194.756721:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1239 00000100:00080000:0.0:1787762194.756734:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762194.756744:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762194.756790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274200448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762194.756829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762194.756834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762194.756854:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762194.756860:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1239 00000100:00080000:0.0:1787762194.756866:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762194.756872:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762194.756886:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762194.756905:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274200576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762194.756918:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200448:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762194.757011:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762194.757079:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762194/real 1787762194] req@ffff9ac451f87c00 x1874603274200448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762194.757107:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274200448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762194.757122:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762194.757128:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762194.757134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200448:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762194.757178:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200576:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762194.757214:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762194.757232:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762194/real 1787762194] req@ffff9ac451f84b40 x1874603274200576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762194.757249:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274200576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762194.757260:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762194.757265:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762194.757269:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200576:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762195.332788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762195.780674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762196.356766:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762196.804663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762197.380757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762197.828683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762198.404765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762198.852684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762199.428768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762199.876662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762199.876671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762199.876680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762199.876689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762199.876704:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1244 00000100:00080000:0.0:1787762199.876723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762199.876734:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762199.876792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274200704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762199.876842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762199.876845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762199.876847:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762199.876851:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1244 00000100:00080000:0.0:1787762199.876855:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762199.876859:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762199.876870:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762199.876898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274200832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762199.876942:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200704:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762199.877054:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200832:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762200.452706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762200.900672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762201.476750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762201.924672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762202.500734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762202.948679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:3.0:1787762202.948889:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:3.0:1787762202.948945:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787762202.948969:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:3.0:1787762202.949019:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787762202.949136:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762199/real 1787762202] req@ffff9ac451f84b40 x1874603274200704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762202.949191:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274200704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762202.949214:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762202.949224:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762202.949232:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200704:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762202.949249:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762199/real 1787762202] req@ffff9ac451f87c00 x1874603274200832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762202.949317:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274200832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762202.949334:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762202.949341:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762202.949348:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274200832:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762203.524743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762203.972633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762204.548686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762204.996604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762204.996626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762204.996635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762204.996640:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762204.996650:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1249 00000100:00080000:0.0:1787762204.996663:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762204.996672:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762204.996715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274200960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762204.996741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762204.996744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762204.996745:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762204.996749:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1249 00000100:00080000:0.0:1787762204.996751:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762204.996754:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762204.996765:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762204.996774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274201088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762204.996809:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200960:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762204.996893:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762204.996937:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762204/real 1787762204] req@ffff9ac451f870c0 x1874603274200960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762204.996958:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274200960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762204.996971:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762204.996976:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762204.996981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274200960:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762204.996990:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201088:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762204.997013:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762204.997036:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762204/real 1787762204] req@ffff9ac451f84000 x1874603274201088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762204.997049:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274201088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762204.997073:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762204.997077:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762204.997091:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201088:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762205.572677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762206.020634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762206.596674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762207.044605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762207.621035:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762208.068640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762208.644798:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762209.092668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762209.668726:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762210.116645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762210.116678:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762210.116689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762210.116695:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762210.116708:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1254 00000100:00080000:1.0:1787762210.116721:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762210.116730:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762210.116776:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274201216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762210.116818:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762210.116823:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762210.116826:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762210.116832:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1254 00000100:00080000:1.0:1787762210.116836:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762210.116841:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762210.116854:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762210.116872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274201344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762210.116877:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201216:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762210.117001:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762210.117062:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762210/real 1787762210] req@ffff9ac447840000 x1874603274201216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762210.117079:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274201216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762210.117105:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762210.117109:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762210.117113:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201216:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762210.117139:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201344:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762210.117161:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762210.117171:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762210/real 1787762210] req@ffff9ac447842580 x1874603274201344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762210.117179:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274201344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762210.117186:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762210.117188:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762210.117191:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201344:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762210.692743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762211.140629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762211.716657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762212.164599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762212.740658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762213.188578:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762213.764692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762214.212599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762214.788640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762215.236686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762215.236720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762215.236730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762215.236755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762215.236768:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1259 00000100:00080000:1.0:1787762215.236784:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762215.236793:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762215.236839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274201472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762215.236881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762215.236889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762215.236893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762215.236902:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1259 00000100:00080000:1.0:1787762215.236908:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762215.236917:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762215.236934:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762215.236959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274201600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762215.236975:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201472:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762215.237066:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762215.237116:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762215/real 1787762215] req@ffff9ac447843840 x1874603274201472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762215.237137:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274201472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762215.237150:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762215.237155:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762215.237160:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201472:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762215.237201:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201600:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762215.237251:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762215.237268:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762215/real 1787762215] req@ffff9ac447841a40 x1874603274201600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762215.237284:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274201600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762215.237295:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762215.237299:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762215.237303:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201600:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762215.812783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762216.260619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762216.836683:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762217.284656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762217.860852:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762218.308637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762218.884726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762219.332663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762219.908752:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762220.356660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762220.356680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762220.356690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762220.356696:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762220.356710:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1265 00000100:00080000:1.0:1787762220.356724:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762220.356733:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762220.356778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274201728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762220.356814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762220.356833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762220.356836:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762220.356842:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1265 00000100:00080000:1.0:1787762220.356847:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762220.356852:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762220.356864:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762220.356883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274201856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762220.356916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201728:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762220.357012:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762220.357071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762220/real 1787762220] req@ffff9ac447841e00 x1874603274201728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762220.357096:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274201728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762220.357107:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762220.357111:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762220.357115:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201728:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762220.357142:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201856:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762220.357162:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762220.357171:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762220/real 1787762220] req@ffff9ac4478412c0 x1874603274201856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762220.357196:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274201856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762220.357202:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762220.357204:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762220.357207:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274201856:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762220.932732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762221.380652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762221.956749:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762222.404622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762222.980728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762223.428663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762224.004741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762224.452593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762225.028655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762225.476593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762225.476651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762225.476659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762225.476664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762225.476677:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1270 00000100:00080000:0.0:1787762225.476693:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762225.476703:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762225.476756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274201984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762225.476793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762225.476797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762225.476799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762225.476803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1270 00000100:00080000:0.0:1787762225.476807:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762225.476812:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762225.476826:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762225.476840:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201984:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762225.476841:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274202112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762225.476940:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762225.476991:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762225/real 1787762225] req@ffff9ac451f861c0 x1874603274201984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762225.477010:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274201984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762225.477022:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762225.477026:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762225.477030:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274201984:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762225.477059:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202112:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762225.477080:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762225.477091:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762225/real 1787762225] req@ffff9ac451f84780 x1874603274202112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762225.477102:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274202112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762225.477120:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762225.477124:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762225.477127:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202112:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762226.052647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762226.500614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762227.076659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762227.524612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762228.100654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762228.548596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762229.124654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762229.572631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762230.148636:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787762230.596696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762230.596702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762230.596714:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762230.596721:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762230.596736:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1275 00000100:00080000:2.0:1787762230.596750:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762230.596760:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762230.596808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274202240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762230.596869:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762230.596875:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762230.596877:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762230.596883:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1275 00000100:00080000:2.0:1787762230.596889:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762230.596897:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762230.596912:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762230.596931:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274202368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762230.596939:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274202240:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762230.597064:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762230.597114:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762230/real 1787762230] req@ffff9ac4523bc3c0 x1874603274202240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762230.597134:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274202240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762230.597148:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762230.597153:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762230.597158:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274202240:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762230.597190:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202368:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762230.597213:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762230.597232:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762230/real 1787762230] req@ffff9ac4523be940 x1874603274202368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762230.597241:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274202368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762230.597247:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762230.597250:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762230.597252:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202368:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762231.172649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762231.620611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762232.196644:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762232.644606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762233.220717:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762233.668625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762234.244738:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762234.692619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762235.268734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762235.716633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762235.716650:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762235.716662:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762235.716668:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762235.716681:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1280 00000100:00080000:2.0:1787762235.716695:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762235.716704:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762235.716751:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274202496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762235.716789:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762235.716794:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762235.716797:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762235.716802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1280 00000100:00080000:2.0:1787762235.716806:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762235.716810:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762235.716842:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762235.716859:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274202624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762235.716940:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274202496:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762235.717025:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762235.717075:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762235/real 1787762235] req@ffff9ac4523bc000 x1874603274202496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762235.717098:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274202496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762235.717113:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762235.717118:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762235.717124:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274202496:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762235.717133:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202624:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762235.717159:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762235.717173:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762235/real 1787762235] req@ffff9ac4523bd680 x1874603274202624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762235.717187:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274202624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762235.717198:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762235.717202:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762235.717206:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202624:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762236.292760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762236.740619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762237.316746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762237.764650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762238.340715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762238.788598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762239.364745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762239.812603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762240.388756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787762240.836644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762240.836656:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762240.836663:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787762240.836665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762240.836680:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1285 00000100:00080000:2.0:1787762240.836695:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762240.836707:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762240.836754:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274202752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762240.836791:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762240.836796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762240.836799:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762240.836804:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1285 00000100:00080000:2.0:1787762240.836808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762240.836812:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762240.836824:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762240.836840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274202880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762240.836894:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274202752:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762240.837002:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762240.837054:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762240/real 1787762240] req@ffff9ac4523bc780 x1874603274202752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762240.837096:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274202752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762240.837111:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762240.837116:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762240.837121:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274202752:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762240.837131:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202880:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762240.837161:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762240.837175:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762240/real 1787762240] req@ffff9ac4523be580 x1874603274202880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762240.837191:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274202880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762240.837201:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762240.837204:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762240.837208:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274202880:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762241.412652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762241.860617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762242.436640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762242.884594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762243.460686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762243.908678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762244.484815:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762244.932630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762245.508768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762245.956665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762245.956679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762245.956685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762245.956687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762245.956701:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1290 00000100:00080000:0.0:1787762245.956721:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762245.956731:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762245.956784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274203008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762245.956834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762245.956841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762245.956845:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762245.956867:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1290 00000100:00080000:0.0:1787762245.956895:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762245.956904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762245.956920:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762245.956942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274203136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762245.956980:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203008:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762245.957089:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762245.957140:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762245/real 1787762245] req@ffff9ac451f85680 x1874603274203008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762245.957164:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274203008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762245.957197:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762245.957203:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762245.957209:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203008:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762245.957219:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203136:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762245.957252:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762245.957302:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762245/real 1787762245] req@ffff9ac451f843c0 x1874603274203136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762245.957318:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274203136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762245.957330:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762245.957334:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762245.957339:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203136:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762246.532708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762246.980623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762247.556686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762248.004648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762248.580761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762249.028629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762249.604722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762250.052642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762250.628786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762251.076641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762251.076653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762251.076660:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762251.076660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762251.076693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1295 00000100:00080000:1.0:1787762251.076707:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762251.076716:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762251.076762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274203264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762251.076801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762251.076824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762251.076827:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762251.076834:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1295 00000100:00080000:1.0:1787762251.076842:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762251.076849:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762251.076863:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762251.076865:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203264:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762251.076888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274203392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762251.076965:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762251.077019:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762251/real 1787762251] req@ffff9ac447843480 x1874603274203264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762251.077042:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274203264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762251.077054:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762251.077058:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762251.077061:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203264:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762251.077088:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203392:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762251.077110:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762251.077120:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762251/real 1787762251] req@ffff9ac4478412c0 x1874603274203392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762251.077128:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274203392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762251.077134:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762251.077137:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762251.077155:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203392:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762251.652767:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762252.100626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762252.676763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762253.124635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762253.700666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762254.148580:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762254.724704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762255.172618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762255.748638:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762256.196628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762256.196649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762256.196657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762256.196663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762256.196676:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1300 00000100:00080000:0.0:1787762256.196691:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762256.196701:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762256.196757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274203520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762256.196792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762256.196796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762256.196798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762256.196802:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1300 00000100:00080000:0.0:1787762256.196806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762256.196810:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762256.196823:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762256.196840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274203648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762256.196841:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203520:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762256.196937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762256.196987:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762256/real 1787762256] req@ffff9ac451f843c0 x1874603274203520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762256.197015:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274203520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762256.197026:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762256.197030:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762256.197035:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203520:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762256.197061:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203648:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762256.197082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762256.197093:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762256/real 1787762256] req@ffff9ac451f85680 x1874603274203648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762256.197104:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274203648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762256.197133:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762256.197136:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762256.197139:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203648:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762256.772698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762257.220595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762257.796681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762258.244610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762258.820740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762259.268659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762259.844706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762260.292611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762260.868780:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762261.316617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762261.316703:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762261.316715:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762261.316721:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762261.316734:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1306 00000100:00080000:1.0:1787762261.316747:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762261.316756:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762261.316804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274203776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762261.316846:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762261.316852:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762261.316855:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762261.316863:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1306 00000100:00080000:1.0:1787762261.316871:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762261.316881:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762261.316898:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762261.316910:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203776:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762261.316931:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274203904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762261.317010:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762261.317061:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762261/real 1787762261] req@ffff9ac447841e00 x1874603274203776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762261.317098:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274203776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762261.317129:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762261.317134:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762261.317140:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274203776:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762261.317198:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203904:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762261.317234:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762261.317252:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762261/real 1787762261] req@ffff9ac447841a40 x1874603274203904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762261.317268:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274203904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762261.317280:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762261.317284:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762261.317288:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274203904:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762261.892679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762262.340626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762262.916737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762263.364614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762263.940775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762264.388630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762264.964681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762265.412647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762265.988733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762266.436676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762266.436677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762266.436693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762266.436700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762266.436714:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1311 00000100:00080000:3.0:1787762266.436733:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762266.436744:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762266.436798:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274204032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762266.436827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762266.436831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762266.436832:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762266.436836:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1311 00000100:00080000:3.0:1787762266.436839:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762266.436842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762266.436853:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762266.436888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274204160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762266.436916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204032:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762266.437024:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204160:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762267.012771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762267.460650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762268.036713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762268.484660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762269.060691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762269.508615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762269.508895:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762269.508945:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762269.508982:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762269.509034:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762269.509196:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762266/real 1787762269] req@ffff9ac451c50b40 x1874603274204032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762269.509236:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274204032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762269.509259:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762269.509269:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762269.509295:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204032:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762269.509311:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762266/real 1787762269] req@ffff9ac451c52940 x1874603274204160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762269.509338:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274204160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762269.509355:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762269.509362:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762269.509369:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204160:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762270.084712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762270.532622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762271.108770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762271.556685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762271.556697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762271.556703:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762271.556703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762271.556720:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1316 00000100:00080000:0.0:1787762271.556736:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762271.556748:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762271.556795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274204288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762271.556831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762271.556835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762271.556838:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762271.556843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1316 00000100:00080000:0.0:1787762271.556847:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762271.556854:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762271.556868:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762271.556910:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274204416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762271.556915:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204288:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762271.557041:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762271.557142:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762271/real 1787762271] req@ffff9ac451f84780 x1874603274204288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762271.557181:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274204288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762271.557204:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762271.557213:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762271.557221:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204288:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762271.557264:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204416:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762271.557316:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762271.557343:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762271/real 1787762271] req@ffff9ac451f861c0 x1874603274204416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762271.557371:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274204416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762271.557389:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762271.557396:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762271.557402:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204416:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762272.132701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762272.580664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762273.156735:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762273.604605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762274.181049:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762274.628658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762275.204765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762275.652650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762276.228736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762276.676664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762276.676672:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762276.676683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762276.676689:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762276.676701:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1321 00000100:00080000:0.0:1787762276.676716:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762276.676726:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762276.676771:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274204544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762276.676808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762276.676812:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762276.676815:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762276.676820:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1321 00000100:00080000:0.0:1787762276.676824:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762276.676829:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762276.676841:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762276.676859:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274204672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762276.676873:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204544:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762276.676976:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762276.677025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762276/real 1787762276] req@ffff9ac451f84000 x1874603274204544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762276.677073:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274204544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762276.677087:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762276.677092:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762276.677098:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204544:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762276.677127:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204672:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762276.677156:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762276.677171:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762276/real 1787762276] req@ffff9ac451f870c0 x1874603274204672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762276.677187:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274204672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762276.677198:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762276.677202:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762276.677206:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204672:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762277.252734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762277.700642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762278.276807:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762278.724619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762279.300990:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762279.748651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762280.324791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762280.772676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762281.348798:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762281.796684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762281.796692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762281.796698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762281.796709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762281.796723:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1326 00000100:00080000:0.0:1787762281.796736:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762281.796774:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762281.796818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274204800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762281.796862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762281.796867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762281.796870:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762281.796875:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1326 00000100:00080000:0.0:1787762281.796881:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762281.796885:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762281.796897:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762281.796912:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274204928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762281.796957:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204800:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762281.797081:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762281.797135:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762281/real 1787762281] req@ffff9ac451f87c00 x1874603274204800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762281.797181:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274204800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762281.797194:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762281.797200:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762281.797205:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274204800:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762281.797215:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204928:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762281.797247:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762281.797262:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762281/real 1787762281] req@ffff9ac451f84b40 x1874603274204928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762281.797277:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274204928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762281.797288:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762281.797292:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762281.797296:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274204928:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762282.372801:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762282.820653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762283.396732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762283.844642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762284.420758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762284.868648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762285.444765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762285.892652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762286.468747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762286.916655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762286.916734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762286.916748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762286.916755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762286.916771:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1331 00000100:00080000:3.0:1787762286.916789:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762286.916801:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762286.916865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274205056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762286.916935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762286.916943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762286.916947:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762286.916972:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1331 00000100:00080000:3.0:1787762286.916988:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762286.916994:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762286.917008:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762286.917025:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762286.917027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274205184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762286.917106:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762286.917149:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762286/real 1787762286] req@ffff9ac451c52d00 x1874603274205056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762286.917165:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274205056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762286.917185:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762286.917188:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762286.917191:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762286.917223:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205184:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762286.917240:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762286.917249:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762286/real 1787762286] req@ffff9ac451c530c0 x1874603274205184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762286.917257:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274205184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762286.917263:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762286.917265:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762286.917267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205184:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762287.492736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762287.940644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762288.516734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762288.964663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762289.540762:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762289.988659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762290.564747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762291.012699:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762291.588712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762292.036654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762292.036689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762292.036700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762292.036707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762292.036718:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1336 00000100:00080000:3.0:1787762292.036732:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762292.036740:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762292.036791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274205312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762292.036820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762292.036823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762292.036844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762292.036849:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1336 00000100:00080000:3.0:1787762292.036852:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762292.036856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762292.036867:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762292.036880:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205312:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762292.036881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274205440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762292.036983:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762292.037030:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762292/real 1787762292] req@ffff9ac451c530c0 x1874603274205312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762292.037052:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274205312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762292.037066:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762292.037071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762292.037076:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205312:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762292.037115:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205440:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762292.037142:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762292.037158:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762292/real 1787762292] req@ffff9ac451c52d00 x1874603274205440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762292.037171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274205440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762292.037178:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762292.037181:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762292.037183:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205440:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762292.612697:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762293.060619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762293.636782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762294.084586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762294.660669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762295.108629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762295.684748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762296.132674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762296.708734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762297.156639:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762297.156652:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762297.156658:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762297.156671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762297.156672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1341 00000100:00080000:3.0:1787762297.156691:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762297.156705:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762297.156768:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274205568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762297.156803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762297.156808:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762297.156810:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762297.156815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1341 00000100:00080000:3.0:1787762297.156820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762297.156846:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762297.156862:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762297.156873:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205568:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762297.156898:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274205696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762297.156990:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762297.157043:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762297/real 1787762297] req@ffff9ac451c52940 x1874603274205568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762297.157067:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274205568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762297.157081:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762297.157099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762297.157105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205568:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762297.157137:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205696:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762297.157166:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762297.157180:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762297/real 1787762297] req@ffff9ac451c50b40 x1874603274205696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762297.157210:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274205696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762297.157221:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762297.157225:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762297.157229:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205696:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762297.732771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762298.180607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762298.756723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762299.204648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762299.780777:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762300.228653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762300.804755:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762301.252687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762301.828730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762302.276631:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762302.276646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762302.276655:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762302.276656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762302.276672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1347 00000100:00080000:3.0:1787762302.276690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762302.276702:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762302.276756:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274205824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762302.276790:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762302.276795:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762302.276797:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762302.276802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1347 00000100:00080000:3.0:1787762302.276807:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762302.276811:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762302.276825:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762302.276839:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274205952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762302.276892:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205824:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762302.276971:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762302.277015:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762302/real 1787762302] req@ffff9ac451c50780 x1874603274205824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762302.277032:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274205824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762302.277042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762302.277045:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762302.277049:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274205824:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762302.277073:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205952:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762302.277088:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762302.277097:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762302/real 1787762302] req@ffff9ac451c503c0 x1874603274205952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762302.277105:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274205952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762302.277122:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762302.277125:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762302.277128:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274205952:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762302.852770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762303.300649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762303.876755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762304.324651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762304.900749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762305.348639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762305.924767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762306.372681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762306.948745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762307.396652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762307.396657:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762307.396672:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762307.396678:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762307.396690:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1352 00000100:00080000:3.0:1787762307.396705:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762307.396713:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762307.396763:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274206080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762307.396792:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762307.396796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762307.396798:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762307.396804:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1352 00000100:00080000:3.0:1787762307.396808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762307.396812:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762307.396825:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762307.396854:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274206208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762307.396864:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206080:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762307.396949:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762307.396993:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762307/real 1787762307] req@ffff9ac451c52580 x1874603274206080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762307.397008:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274206080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762307.397017:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762307.397022:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762307.397027:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206080:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762307.397061:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206208:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762307.397085:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762307.397095:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762307/real 1787762307] req@ffff9ac451c50f00 x1874603274206208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762307.397104:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274206208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762307.397111:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762307.397113:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762307.397116:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206208:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762307.972748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762308.420641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762308.996780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762309.444635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762310.020783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762310.468670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762311.044778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762311.492701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762312.068771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762312.516641:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762312.516654:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762312.516660:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762312.516664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762312.516674:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1357 00000100:00080000:3.0:1787762312.516690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762312.516700:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762312.516764:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274206336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762312.516821:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762312.516827:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762312.516829:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762312.516835:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1357 00000100:00080000:3.0:1787762312.516839:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762312.516845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762312.516858:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762312.516877:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274206464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762312.516898:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206336:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762312.516989:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762312.517056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762312/real 1787762312] req@ffff9ac451c50f00 x1874603274206336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762312.517091:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274206336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762312.517106:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762312.517112:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762312.517118:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762312.517150:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206464:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762312.517184:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762312.517201:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762312/real 1787762312] req@ffff9ac451c52580 x1874603274206464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762312.517216:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274206464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762312.517228:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762312.517232:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762312.517236:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206464:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762313.092744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762313.540655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762314.116785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762314.564629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762315.140813:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762315.588653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762316.164742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762316.612655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762317.188820:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762317.636672:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762317.636679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762317.636702:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762317.636708:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762317.636720:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1362 00000100:00080000:3.0:1787762317.636736:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762317.636746:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762317.636829:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274206592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762317.636881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762317.636889:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762317.636893:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762317.636902:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1362 00000100:00080000:3.0:1787762317.636911:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762317.636919:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762317.636944:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762317.636967:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274206720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762317.637017:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206592:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762317.637101:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762317.637151:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762317/real 1787762317] req@ffff9ac451c503c0 x1874603274206592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762317.637188:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274206592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762317.637203:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762317.637209:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762317.637214:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206592:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762317.637223:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206720:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762317.637254:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762317.637269:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762317/real 1787762317] req@ffff9ac451c50780 x1874603274206720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762317.637285:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274206720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762317.637295:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762317.637299:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762317.637304:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206720:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762318.212741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762318.660655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762319.236744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762319.684674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762320.260698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762320.708648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762321.284711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762321.732671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762322.308728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762322.756628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762322.756645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762322.756655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762322.756661:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762322.756675:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1367 00000100:00080000:3.0:1787762322.756708:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762322.756718:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762322.756793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274206848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762322.756843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762322.756851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762322.756855:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762322.756863:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1367 00000100:00080000:3.0:1787762322.756871:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762322.756878:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762322.756892:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762322.756932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274206976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762322.756949:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206848:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762322.757042:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762322.757092:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762322/real 1787762322] req@ffff9ac451c50b40 x1874603274206848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762322.757116:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274206848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762322.757150:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762322.757156:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762322.757161:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274206848:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762322.757192:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206976:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762322.757224:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762322.757240:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762322/real 1787762322] req@ffff9ac451c52940 x1874603274206976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762322.757255:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274206976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762322.757266:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762322.757270:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762322.757274:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274206976:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762323.332706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762323.780629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762324.356807:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762324.804625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762325.380763:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762325.828655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762326.404804:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762326.852647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762327.428668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762327.876600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762327.876627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762327.876633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762327.876638:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762327.876645:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1372 00000100:00080000:3.0:1787762327.876654:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762327.876660:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762327.876692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274207104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762327.876743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762327.876746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762327.876748:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762327.876752:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1372 00000100:00080000:3.0:1787762327.876755:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762327.876758:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762327.876767:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762327.876777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274207232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762327.876789:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207104:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762327.876843:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762327.876879:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762327/real 1787762327] req@ffff9ac451c52d00 x1874603274207104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762327.876898:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274207104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762327.876910:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762327.876915:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762327.876919:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762327.876955:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274207232:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762327.876979:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762327.876989:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762327/real 1787762327] req@ffff9ac451c530c0 x1874603274207232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762327.876998:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274207232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762327.877004:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762327.877007:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762327.877009:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274207232:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762328.452641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762328.900604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762329.476698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762329.924616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762330.500702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762330.948588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762331.524725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762331.972646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762332.548798:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762332.996604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762332.996643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762332.996656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762332.996683:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762332.996698:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1377 00000100:00080000:3.0:1787762332.996714:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762332.996725:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762332.996784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274207360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762332.996823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762332.996828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762332.996848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762332.996856:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1377 00000100:00080000:3.0:1787762332.996862:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762332.996869:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762332.996886:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762332.996906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207360:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762332.996909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274207488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762332.997094:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274207488:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762333.572754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762334.020717:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762334.596681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762335.044638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762335.620729:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762336.068764:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762336.068964:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762336.069014:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762336.069031:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762336.069077:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762336.069129:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762332/real 1787762336] req@ffff9ac451c52d00 x1874603274207488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762336.069161:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274207488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762336.069176:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762336.069182:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762336.069187:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274207488:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787762336.069197:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762332/real 1787762336] req@ffff9ac451c530c0 x1874603274207360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762336.069213:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274207360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762336.069252:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762336.069257:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762336.069261:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207360:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787762336.644735:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762337.092851:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762337.668673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762338.116640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762338.116653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762338.116657:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762338.116670:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1382 00000100:00080000:0.0:1787762338.116683:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762338.116691:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762338.116714:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787762338.116741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274207616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762338.116776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762338.116778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762338.116780:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762338.116784:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1382 00000100:00080000:0.0:1787762338.116788:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762338.116794:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762338.116807:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762338.116820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207616:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762338.116825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274207744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762338.116915:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762338.116963:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762338/real 1787762338] req@ffff9ac451f84b40 x1874603274207616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762338.116983:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274207616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762338.117019:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762338.117025:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762338.117030:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207616:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762338.117062:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274207744:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762338.117092:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762338.117108:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762338/real 1787762338] req@ffff9ac451f87c00 x1874603274207744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762338.117122:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274207744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762338.117131:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762338.117135:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762338.117139:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274207744:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762338.692788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762339.140804:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762339.716743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762340.164691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762340.741465:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762341.188707:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762341.764764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762342.212622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762342.788660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762343.236654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762343.236697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762343.236705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762343.236709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762343.236718:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1387 00000100:00080000:3.0:1787762343.236729:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762343.236736:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762343.236791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274207872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762343.236827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762343.236846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762343.236848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762343.236852:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1387 00000100:00080000:3.0:1787762343.236856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762343.236859:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762343.236869:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762343.236881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274208000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762343.236920:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207872:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762343.236984:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762343.237041:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762343/real 1787762343] req@ffff9ac451c52940 x1874603274207872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762343.237057:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274207872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762343.237068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762343.237071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762343.237075:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274207872:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762343.237103:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208000:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762343.237125:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762343.237136:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762343/real 1787762343] req@ffff9ac451c50b40 x1874603274208000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762343.237147:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274208000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762343.237155:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762343.237157:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762343.237160:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208000:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762343.812657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762344.260679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762344.836663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762345.284601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762345.861048:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762346.308688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762346.884708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762347.332614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762347.908680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762348.356609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762348.356620:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762348.356628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762348.356632:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762348.356643:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1393 00000100:00080000:3.0:1787762348.356653:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762348.356662:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762348.356700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274208128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762348.356731:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762348.356735:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762348.356737:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762348.356742:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1393 00000100:00080000:3.0:1787762348.356761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762348.356767:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762348.356778:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762348.356791:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208128:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762348.356796:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274208256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762348.356848:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762348.356881:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762348/real 1787762348] req@ffff9ac451c50780 x1874603274208128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762348.356894:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274208128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762348.356902:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762348.356905:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762348.356908:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208128:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762348.356924:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208256:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762348.356938:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762348.356946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762348/real 1787762348] req@ffff9ac451c503c0 x1874603274208256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762348.356961:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274208256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762348.356967:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762348.356969:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762348.356971:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208256:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762348.932665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762349.380689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762349.956715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762350.404603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762350.980648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762351.428658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762352.004644:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762352.452641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762353.028675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762353.476594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762353.476695:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762353.476708:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762353.476715:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762353.476729:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1398 00000100:00080000:3.0:1787762353.476746:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762353.476757:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762353.476813:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274208384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762353.476851:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762353.476857:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762353.476859:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762353.476865:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1398 00000100:00080000:3.0:1787762353.476869:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762353.476873:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762353.476887:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762353.476922:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274208512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762353.476977:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208384:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762353.477078:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762353.477154:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762353/real 1787762353] req@ffff9ac451c52580 x1874603274208384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762353.477180:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274208384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762353.477195:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762353.477200:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762353.477206:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208384:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762353.477216:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208512:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762353.477244:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762353.477259:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762353/real 1787762353] req@ffff9ac451c50f00 x1874603274208512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762353.477274:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274208512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762353.477315:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762353.477324:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762353.477331:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208512:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762354.052680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762354.500612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762355.076692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762355.524663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762356.100635:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762356.548609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762357.124648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762357.572646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762358.148646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762358.596615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762358.596618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762358.596625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762358.596644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762358.596652:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1403 00000100:00080000:0.0:1787762358.596662:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762358.596670:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762358.596706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274208640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762358.596743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762358.596748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762358.596750:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762358.596756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1403 00000100:00080000:0.0:1787762358.596760:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762358.596766:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762358.596775:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762358.596787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274208768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762358.596795:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208640:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762358.596853:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762358.596886:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762358/real 1787762358] req@ffff9ac451f870c0 x1874603274208640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762358.596899:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274208640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762358.596907:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762358.596910:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762358.596913:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208640:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762358.596930:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208768:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762358.596944:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762358.596952:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762358/real 1787762358] req@ffff9ac451f84000 x1874603274208768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762358.596960:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274208768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762358.596966:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762358.596968:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762358.596970:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274208768:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762359.172658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762359.620625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762360.196657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762360.644603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762361.220671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762361.668610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762362.244667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762362.692610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762363.268659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762363.716601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762363.716627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762363.716633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762363.716637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762363.716645:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1408 00000100:00080000:3.0:1787762363.716654:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762363.716660:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762363.716693:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274208896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762363.716717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762363.716721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762363.716724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762363.716728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1408 00000100:00080000:3.0:1787762363.716731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762363.716736:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762363.716762:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762363.716771:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762363.716774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274209024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762363.716835:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762363.716871:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762363/real 1787762363] req@ffff9ac451c50f00 x1874603274208896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762363.716884:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274208896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762363.716892:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762363.716895:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762363.716898:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274208896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762363.716917:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209024:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762363.716934:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762363.716941:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762363/real 1787762363] req@ffff9ac451c52580 x1874603274209024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762363.716949:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274209024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762363.716955:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762363.716957:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762363.716959:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209024:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762364.292633:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762364.740583:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762365.316730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762365.764659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762366.340682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762366.788608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762367.364664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762367.812602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762368.388702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762368.836655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762368.836664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762368.836669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762368.836680:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1413 00000100:00100000:2.0:1787762368.836681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762368.836693:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762368.836703:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762368.836740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274209152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762368.836767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762368.836785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762368.836787:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762368.836791:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1413 00000100:00080000:0.0:1787762368.836795:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762368.836798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762368.836807:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762368.836818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274209280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762368.836878:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209152:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762368.836963:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762368.837017:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762368/real 1787762368] req@ffff9ac451f861c0 x1874603274209152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762368.837055:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274209152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762368.837070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762368.837075:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762368.837080:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209152:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762368.837090:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209280:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762368.837120:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762368.837134:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762368/real 1787762368] req@ffff9ac451f84780 x1874603274209280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762368.837149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274209280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762368.837160:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762368.837164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762368.837168:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209280:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762369.412656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762369.860604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762370.436627:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762370.884603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762371.460672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762371.908588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762372.484658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762372.932611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762373.508742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762373.956674:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762373.956675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762373.956691:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762373.956699:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762373.956712:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1418 00000100:00080000:1.0:1787762373.956727:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762373.956738:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762373.956786:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274209408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762373.956849:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762373.956854:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762373.956859:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762373.956866:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1418 00000100:00080000:1.0:1787762373.956874:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762373.956880:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762373.956894:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762373.956903:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209408:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762373.956917:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274209536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762373.957015:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762373.957071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762373/real 1787762373] req@ffff9ac447843840 x1874603274209408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762373.957088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274209408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762373.957098:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762373.957101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762373.957105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209408:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762373.957132:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209536:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762373.957151:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762373.957160:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762373/real 1787762373] req@ffff9ac447842580 x1874603274209536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762373.957168:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274209536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762373.957190:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762373.957193:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762373.957195:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209536:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762374.532655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762374.980617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762375.556664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762376.004599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762376.580783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762377.028676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762377.604778:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762378.052653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762378.628783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762379.076644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762379.076724:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762379.076734:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762379.076739:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762379.076751:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1423 00000100:00080000:3.0:1787762379.076765:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762379.076774:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762379.076818:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274209664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762379.076854:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762379.076859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762379.076879:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762379.076887:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1423 00000100:00080000:3.0:1787762379.076893:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762379.076900:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762379.076913:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762379.076934:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274209792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762379.076948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209664:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762379.077055:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762379.077111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762379/real 1787762379] req@ffff9ac451c503c0 x1874603274209664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762379.077131:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274209664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762379.077143:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762379.077148:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762379.077152:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762379.077182:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209792:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762379.077202:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762379.077213:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762379/real 1787762379] req@ffff9ac451c50780 x1874603274209792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762379.077223:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274209792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762379.077231:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762379.077234:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762379.077237:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274209792:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762379.652697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762380.100654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762380.676800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762381.124719:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762381.700764:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762382.148638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762382.724740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762383.172598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762383.748630:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762384.196600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762384.196601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762384.196612:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762384.196617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762384.196629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1428 00000100:00080000:3.0:1787762384.196642:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762384.196651:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762384.196693:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274209920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762384.196720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762384.196723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762384.196725:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762384.196728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1428 00000100:00080000:3.0:1787762384.196730:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762384.196741:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762384.196758:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762384.196769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274210048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762384.196878:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209920:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762384.196987:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762384.197045:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762384/real 1787762384] req@ffff9ac451c50b40 x1874603274209920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762384.197071:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274209920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762384.197086:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762384.197091:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762384.197097:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274209920:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762384.197106:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210048:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762384.197132:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762384.197145:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762384/real 1787762384] req@ffff9ac451c52940 x1874603274210048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762384.197160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274210048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762384.197170:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762384.197174:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762384.197178:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210048:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762384.772650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762385.220593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762385.796666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762386.244627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762386.820699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762387.268602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762387.844661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762388.292588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762388.868678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762389.316614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762389.316621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762389.316626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762389.316632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762389.316642:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1434 00000100:00080000:3.0:1787762389.316655:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762389.316663:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762389.316708:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274210176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762389.316734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762389.316737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762389.316739:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762389.316742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1434 00000100:00080000:3.0:1787762389.316744:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762389.316746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762389.316756:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762389.316765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274210304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762389.316867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210176:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762389.316961:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762389.317018:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762389/real 1787762389] req@ffff9ac451c530c0 x1874603274210176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762389.317080:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274210176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762389.317102:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762389.317111:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762389.317120:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210176:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762389.317136:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210304:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762389.317178:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762389.317196:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762389/real 1787762389] req@ffff9ac451c52d00 x1874603274210304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762389.317213:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274210304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762389.317225:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762389.317229:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762389.317234:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210304:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762389.892652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762390.340605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762390.916664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762391.364608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762391.940654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762392.388612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762392.964646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762393.412612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762393.988684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762394.436606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762394.436612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762394.436618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762394.436624:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762394.436635:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1439 00000100:00080000:3.0:1787762394.436649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762394.436657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762394.436711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274210432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762394.436753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762394.436757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762394.436760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762394.436765:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1439 00000100:00080000:3.0:1787762394.436770:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762394.436775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762394.436789:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762394.436802:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274210560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762394.436833:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210432:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762394.436913:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762394.436959:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762394/real 1787762394] req@ffff9ac451c52d00 x1874603274210432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762394.436974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274210432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762394.437006:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762394.437009:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762394.437012:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210432:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762394.437018:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210560:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762394.437037:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762394.437045:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762394/real 1787762394] req@ffff9ac451c530c0 x1874603274210560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762394.437054:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274210560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762394.437060:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762394.437062:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762394.437064:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210560:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762395.012651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762395.460620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762396.036687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762396.484603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762397.060689:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762397.508597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762398.084672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762398.532626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762399.108774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762399.556584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762399.556603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762399.556609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762399.556614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762399.556625:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1444 00000100:00080000:3.0:1787762399.556637:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762399.556644:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762399.556705:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274210688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762399.556741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762399.556744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762399.556746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762399.556749:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1444 00000100:00080000:3.0:1787762399.556753:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762399.556756:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762399.556767:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762399.556779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274210816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762399.556810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762399.556923:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210816:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762400.132651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762400.580612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762401.156650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762401.604594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762402.180654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762402.628653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762402.628895:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762402.628947:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762402.628968:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762402.629022:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762402.629174:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762399/real 1787762402] req@ffff9ac451c52940 x1874603274210688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762402.629247:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274210688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762402.629279:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762402.629289:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762402.629320:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762402.629337:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762399/real 1787762402] req@ffff9ac451c50b40 x1874603274210816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762402.629368:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274210816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762402.629386:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762402.629394:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762402.629401:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274210816:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762403.204653:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762403.652632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762404.228763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762404.676644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762404.676656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762404.676674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762404.676681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762404.676695:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1449 00000100:00080000:3.0:1787762404.676712:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762404.676722:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762404.676766:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274210944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762404.676805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762404.676810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762404.676813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762404.676818:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1449 00000100:00080000:3.0:1787762404.676822:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762404.676827:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762404.676839:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762404.676856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274211072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762404.676885:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210944:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762404.676969:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762404.677033:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762404/real 1787762404] req@ffff9ac451c50780 x1874603274210944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762404.677068:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274210944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762404.677088:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762404.677096:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762404.677104:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274210944:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762404.677144:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211072:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762404.677218:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762404.677246:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762404/real 1787762404] req@ffff9ac451c503c0 x1874603274211072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762404.677307:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274211072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762404.677326:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762404.677333:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762404.677340:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211072:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762405.252785:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762405.700663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762406.276750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762406.724614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762407.301246:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762407.748582:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762408.324782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762408.772611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762409.348664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762409.796659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762409.796692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762409.796705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762409.796711:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762409.796725:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1454 00000100:00080000:1.0:1787762409.796738:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762409.796748:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762409.796808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274211200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762409.796844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762409.796849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762409.796868:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762409.796875:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1454 00000100:00080000:1.0:1787762409.796881:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762409.796888:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762409.796902:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762409.796922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274211328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762409.796933:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211200:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762409.797031:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762409.797098:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762409/real 1787762409] req@ffff9ac447840000 x1874603274211200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762409.797125:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274211200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762409.797171:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762409.797177:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762409.797183:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211200:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762409.797213:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211328:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762409.797249:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762409.797267:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762409/real 1787762409] req@ffff9ac447840f00 x1874603274211328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762409.797283:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274211328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762409.797315:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762409.797321:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762409.797325:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211328:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762410.372789:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762410.820624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762411.396739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762411.844620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762412.420793:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762412.868618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762413.444756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762413.892664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762414.468693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762414.916626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762414.916711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762414.916723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762414.916729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762414.916741:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1459 00000100:00080000:1.0:1787762414.916771:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762414.916779:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762414.916812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274211456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762414.916875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762414.916883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762414.916887:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762414.916895:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1459 00000100:00080000:1.0:1787762414.916903:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762414.916911:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762414.916927:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762414.916976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274211584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762414.916987:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211456:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762414.917075:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762414.917128:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762414/real 1787762414] req@ffff9ac447840780 x1874603274211456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762414.917160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274211456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762414.917168:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762414.917171:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762414.917174:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762414.917191:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211584:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762414.917207:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762414.917214:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762414/real 1787762414] req@ffff9ac447841680 x1874603274211584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762414.917222:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274211584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762414.917228:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762414.917230:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762414.917232:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211584:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762415.492770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762415.940617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762416.516721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762416.964668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762417.540723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762417.988687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762418.564719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762419.012630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762419.588775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762420.036658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762420.036671:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762420.036672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762420.036677:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762420.036693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1464 00000100:00080000:1.0:1787762420.036707:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762420.036718:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762420.036767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274211712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762420.036814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762420.036820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762420.036823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762420.036829:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1464 00000100:00080000:1.0:1787762420.036834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762420.036839:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762420.036852:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762420.036872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274211840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762420.036892:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211712:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762420.036999:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762420.037055:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762420/real 1787762420] req@ffff9ac4478403c0 x1874603274211712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762420.037095:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274211712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762420.037108:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762420.037112:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762420.037116:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211712:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762420.037148:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211840:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762420.037173:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762420.037184:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762420/real 1787762420] req@ffff9ac4478421c0 x1874603274211840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762420.037195:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274211840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762420.037204:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762420.037207:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762420.037210:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274211840:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762420.612775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762421.060657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762421.636719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762422.084636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762422.660722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762423.108636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762423.684833:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762424.132664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762424.708700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762425.156643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762425.156655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762425.156659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762425.156661:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762425.156679:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1469 00000100:00080000:3.0:1787762425.156695:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762425.156703:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762425.156752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274211968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762425.156787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762425.156791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762425.156792:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762425.156796:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1469 00000100:00080000:3.0:1787762425.156798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762425.156801:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762425.156812:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762425.156821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274212096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762425.156859:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211968:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762425.156949:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762425.157006:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762425/real 1787762425] req@ffff9ac451c52580 x1874603274211968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762425.157046:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274211968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762425.157057:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762425.157061:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762425.157064:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274211968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762425.157092:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212096:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762425.157112:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762425.157121:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762425/real 1787762425] req@ffff9ac451c50f00 x1874603274212096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762425.157130:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274212096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762425.157136:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762425.157139:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762425.157141:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212096:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762425.732771:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762426.180647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762426.756803:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762427.204627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762427.780791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762428.228630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762428.804696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762429.252666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762429.828672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762430.276589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762430.276597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762430.276606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762430.276610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762430.276621:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1475 00000100:00080000:3.0:1787762430.276632:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762430.276640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762430.276676:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274212224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762430.276699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762430.276701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762430.276703:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762430.276706:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1475 00000100:00080000:3.0:1787762430.276709:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762430.276712:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762430.276720:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762430.276730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274212352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762430.276735:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212224:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762430.276795:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762430.276830:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762430/real 1787762430] req@ffff9ac451c50f00 x1874603274212224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762430.276843:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274212224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762430.276867:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762430.276870:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762430.276873:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212224:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762430.276891:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212352:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762430.276908:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762430.276917:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762430/real 1787762430] req@ffff9ac451c52580 x1874603274212352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762430.276925:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274212352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762430.276930:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762430.276933:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762430.276935:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212352:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762430.852774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762431.300608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762431.876780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762432.324645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762432.900754:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762433.348675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762433.924735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762434.372668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762434.948750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762435.396648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762435.396660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762435.396666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762435.396679:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1480 00000100:00080000:3.0:1787762435.396693:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787762435.396698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762435.396705:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762435.396754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274212480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762435.396791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762435.396796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762435.396798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762435.396803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1480 00000100:00080000:3.0:1787762435.396808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762435.396812:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762435.396824:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762435.396840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274212608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762435.396867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212480:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762435.396951:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762435.397030:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762435/real 1787762435] req@ffff9ac451c503c0 x1874603274212480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762435.397054:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274212480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762435.397068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762435.397073:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762435.397078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212480:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762435.397124:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212608:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762435.397156:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762435.397172:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762435/real 1787762435] req@ffff9ac451c50780 x1874603274212608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762435.397188:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274212608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762435.397199:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762435.397203:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762435.397208:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212608:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762435.972770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762436.420649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762436.996733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762437.444679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762438.020777:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762438.468655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762439.044750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762439.492676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762440.068735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762440.516627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762440.516723:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762440.516732:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762440.516737:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762440.516749:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1485 00000100:00080000:0.0:1787762440.516762:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762440.516771:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762440.516820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274212736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762440.516862:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762440.516867:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762440.516870:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762440.516876:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1485 00000100:00080000:0.0:1787762440.516880:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762440.516911:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762440.516944:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212736:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787762440.516949:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762440.516973:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274212864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762440.517052:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762440.517114:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762440/real 1787762440] req@ffff9ac451f85680 x1874603274212736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762440.517132:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274212736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762440.517157:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762440.517161:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762440.517165:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212736:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762440.517196:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212864:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762440.517218:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762440.517228:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762440/real 1787762440] req@ffff9ac451f843c0 x1874603274212864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762440.517237:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274212864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762440.517243:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762440.517246:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762440.517248:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274212864:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762441.092748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762441.540664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762442.116724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762442.564643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762443.140777:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762443.588632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762444.164745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762444.612604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762445.188718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762445.636594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762445.636599:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762445.636609:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762445.636614:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762445.636625:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1490 00000100:00080000:0.0:1787762445.636637:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762445.636645:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762445.636692:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274212992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762445.636716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762445.636719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762445.636721:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762445.636724:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1490 00000100:00080000:0.0:1787762445.636727:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762445.636730:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762445.636752:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762445.636761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274213120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762445.636779:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212992:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762445.636855:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762445.636914:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762445/real 1787762445] req@ffff9ac451f843c0 x1874603274212992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762445.636946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274212992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762445.636960:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762445.636965:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762445.636970:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274212992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762445.637010:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213120:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762445.637029:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762445.637039:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762445/real 1787762445] req@ffff9ac451f85680 x1874603274213120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762445.637048:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274213120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762445.637055:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762445.637057:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762445.637059:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213120:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762446.212672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762446.660676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762447.236769:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762447.684637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762448.260760:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762448.708630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762449.284774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762449.732590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762450.308726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762450.756628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762450.756677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762450.756689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762450.756695:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762450.756708:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1495 00000100:00080000:2.0:1787762450.756722:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762450.756749:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762450.756818:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274213248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762450.756879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762450.756885:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762450.756887:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762450.756893:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1495 00000100:00080000:2.0:1787762450.756900:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762450.756907:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762450.756921:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762450.756941:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274213376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762450.756954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274213248:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762450.757049:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762450.757114:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762450/real 1787762450] req@ffff9ac4523bcf00 x1874603274213248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762450.757145:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274213248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762450.757159:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762450.757164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762450.757169:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274213248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762450.757200:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213376:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762450.757227:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762450.757242:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762450/real 1787762450] req@ffff9ac4523bcb40 x1874603274213376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762450.757272:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274213376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762450.757283:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762450.757289:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762450.757295:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213376:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762451.332761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762451.780583:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762452.356775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762452.804633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762453.380691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762453.828598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762454.404789:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762454.852675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762455.428771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762455.876614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762455.876626:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762455.876639:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762455.876644:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762455.876658:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1500 00000100:00080000:2.0:1787762455.876672:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762455.876684:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762455.876732:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274213504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762455.876770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762455.876776:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762455.876781:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762455.876790:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1500 00000100:00100000:1.0:1787762455.876827:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274213504:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787762455.876849:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762455.876860:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762455.876875:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762455.876902:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274213632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762455.876936:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762455.876993:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762455/real 1787762455] req@ffff9ac4523be1c0 x1874603274213504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762455.877017:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274213504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762455.877032:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762455.877037:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762455.877043:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274213504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762455.877076:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213632:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762455.877098:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762455.877108:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762455/real 1787762455] req@ffff9ac4523bf0c0 x1874603274213632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762455.877117:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274213632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762455.877143:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762455.877145:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762455.877148:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213632:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762456.452775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762456.900662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762457.476771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762457.924629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762458.500752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762458.948613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762459.524649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762459.972697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762460.548828:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762460.996692:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762460.996705:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762460.996708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762460.996711:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762460.996730:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1505 00000100:00080000:0.0:1787762460.996743:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762460.996754:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762460.996827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274213760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762460.996861:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762460.996866:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762460.996869:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762460.996874:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1505 00000100:00080000:0.0:1787762460.996878:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762460.996884:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762460.996896:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762460.996913:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274213888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762460.996927:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274213760:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762460.997033:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762460.997096:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762460/real 1787762460] req@ffff9ac451f84780 x1874603274213760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762460.997123:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274213760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762460.997138:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762460.997144:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762460.997149:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274213760:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762460.997178:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213888:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762460.997205:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762460.997220:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762460/real 1787762460] req@ffff9ac451f861c0 x1874603274213888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762460.997236:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274213888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762460.997247:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762460.997251:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762460.997255:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274213888:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762461.572722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762462.020641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762462.596780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762463.044667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762463.620686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762464.068643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762464.644733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762465.092649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762465.668728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762466.116607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762466.116659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762466.116667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762466.116672:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762466.116681:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1510 00000100:00080000:1.0:1787762466.116692:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762466.116717:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762466.116804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274214016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762466.116835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762466.116838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762466.116840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762466.116844:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1510 00000100:00080000:1.0:1787762466.116848:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762466.116853:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762466.116863:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762466.116876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274214144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762466.116944:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762466.117111:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214144:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762466.692695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762467.140673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762467.716676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762468.164593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762468.740702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762469.188629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787762469.188940:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787762469.189002:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787762469.189030:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787762469.189098:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762469.189248:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762466/real 1787762469] req@ffff9ac4478430c0 x1874603274214016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762469.189300:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274214016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762469.189323:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762469.189332:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762469.189340:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762469.189356:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762466/real 1787762469] req@ffff9ac447842d00 x1874603274214144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762469.189380:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274214144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762469.189395:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762469.189402:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762469.189408:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214144:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762469.764745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762470.212665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762470.788746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762471.236674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762471.236767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762471.236778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762471.236784:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762471.236816:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1515 00000100:00080000:3.0:1787762471.236832:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762471.236842:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762471.236896:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274214272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762471.236944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762471.236950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762471.236953:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762471.236960:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1515 00000100:00080000:3.0:1787762471.236966:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762471.236971:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762471.236985:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762471.237002:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274214400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762471.237024:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214272:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762471.237133:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762471.237210:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762471/real 1787762471] req@ffff9ac451c50b40 x1874603274214272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762471.237262:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274214272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762471.237278:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762471.237285:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762471.237291:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762471.237329:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214400:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762471.237367:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762471.237385:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762471/real 1787762471] req@ffff9ac451c52940 x1874603274214400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762471.237402:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274214400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762471.237414:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762471.237418:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762471.237423:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214400:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762471.812760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762472.260654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762472.836786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762473.284649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762473.860978:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762474.308629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762474.884657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762475.332624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762475.909059:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762476.356723:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762476.356839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762476.356866:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762476.356873:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762476.356889:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1521 00000100:00080000:0.0:1787762476.356905:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762476.356918:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762476.356975:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274214528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762476.357033:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762476.357038:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762476.357050:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762476.357056:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1521 00000100:00080000:0.0:1787762476.357063:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762476.357069:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762476.357082:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762476.357102:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274214656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762476.357129:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214528:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762476.357229:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762476.357291:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762476/real 1787762476] req@ffff9ac451f84000 x1874603274214528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762476.357373:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274214528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762476.357393:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762476.357402:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762476.357411:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762476.357451:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214656:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762476.357495:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762476.357515:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762476/real 1787762476] req@ffff9ac451f870c0 x1874603274214656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762476.357579:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274214656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762476.357592:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762476.357596:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762476.357600:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214656:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762476.932791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762477.380648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762477.956800:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762478.404632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762478.980665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762479.428626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762480.004823:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762480.452678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762481.028759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762481.476693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762481.476749:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762481.476759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762481.476765:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762481.476777:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1526 00000100:00080000:3.0:1787762481.476791:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762481.476799:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762481.476860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274214784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762481.476902:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762481.476907:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762481.476910:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762481.476915:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1526 00000100:00080000:3.0:1787762481.476920:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762481.476926:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762481.476938:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762481.476954:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274214912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762481.476992:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214784:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762481.477074:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762481.477122:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762481/real 1787762481] req@ffff9ac451c530c0 x1874603274214784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762481.477145:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274214784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762481.477172:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762481.477178:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762481.477184:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274214784:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762481.477194:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214912:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762481.477226:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762481.477242:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762481/real 1787762481] req@ffff9ac451c52d00 x1874603274214912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762481.477258:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274214912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762481.477268:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762481.477273:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762481.477277:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274214912:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762482.052783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762482.500681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762483.076715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762483.524709:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762484.100798:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762484.548643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762485.124670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762485.572658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762486.148803:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762486.596651:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762486.596665:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762486.596670:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762486.596686:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1531 00000100:00100000:2.0:1787762486.596687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762486.596702:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762486.596713:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762486.596765:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274215040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762486.596793:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762486.596797:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762486.596798:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762486.596818:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1531 00000100:00080000:3.0:1787762486.596821:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762486.596825:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762486.596836:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762486.596846:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274215168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762486.596917:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215040:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762486.597012:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762486.597064:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762486/real 1787762486] req@ffff9ac451c52d00 x1874603274215040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762486.597088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274215040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762486.597129:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762486.597135:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762486.597141:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215040:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762486.597151:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215168:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762486.597183:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762486.597199:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762486/real 1787762486] req@ffff9ac451c530c0 x1874603274215168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762486.597214:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274215168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762486.597225:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762486.597229:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762486.597234:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215168:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762487.172759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762487.620668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762488.196659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762488.644629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762489.220748:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762489.668679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762490.244743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762490.692669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762491.268743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762491.716677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762491.716735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762491.716768:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762491.716775:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762491.716792:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1536 00000100:00080000:3.0:1787762491.716810:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762491.716822:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762491.716886:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274215296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762491.716930:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762491.716936:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762491.716939:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762491.716944:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1536 00000100:00080000:3.0:1787762491.716948:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762491.716953:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762491.716969:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762491.717014:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762491.717025:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274215424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762491.717118:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762491.717174:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762491/real 1787762491] req@ffff9ac451c52940 x1874603274215296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762491.717199:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274215296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762491.717214:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762491.717239:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762491.717245:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762491.717279:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215424:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762491.717311:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762491.717327:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762491/real 1787762491] req@ffff9ac451c50b40 x1874603274215424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762491.717343:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274215424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762491.717353:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762491.717358:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762491.717362:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215424:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762492.292730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762492.740678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762493.316760:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762493.764648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762494.340693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762494.788616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762495.364792:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762495.812669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762496.388723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762496.836681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762496.836693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762496.836704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762496.836710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762496.836735:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1541 00000100:00080000:1.0:1787762496.836749:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762496.836759:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762496.836808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274215552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762496.836848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762496.836875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762496.836878:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762496.836884:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1541 00000100:00080000:1.0:1787762496.836890:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762496.836896:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762496.836908:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762496.836924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274215680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762496.836982:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215552:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762496.837064:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762496.837118:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762496/real 1787762496] req@ffff9ac447842940 x1874603274215552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762496.837142:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274215552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762496.837157:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762496.837162:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762496.837168:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215552:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762496.837177:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215680:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762496.837206:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762496.837229:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762496/real 1787762496] req@ffff9ac447840b40 x1874603274215680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762496.837273:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274215680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762496.837291:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762496.837299:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762496.837306:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215680:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762497.412735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762497.860645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762498.436790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762498.884631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762499.460736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762499.908658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762500.484733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762500.932640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762501.508708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762501.956599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762501.956629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762501.956635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762501.956640:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762501.956648:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1546 00000100:00080000:3.0:1787762501.956656:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762501.956663:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762501.956696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274215808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762501.956718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762501.956721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762501.956722:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762501.956725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1546 00000100:00080000:3.0:1787762501.956728:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762501.956730:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762501.956738:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762501.956748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274215936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762501.956769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215808:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762501.956824:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762501.956859:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762501/real 1787762501] req@ffff9ac451c50780 x1874603274215808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762501.956872:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274215808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762501.956880:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762501.956883:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762501.956886:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274215808:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762501.956891:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215936:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762501.956905:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762501.956913:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762501/real 1787762501] req@ffff9ac451c503c0 x1874603274215936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762501.956920:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274215936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762501.956944:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762501.956947:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762501.956949:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274215936:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762502.532751:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762502.980597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762503.556786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762504.004654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762504.580751:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762505.028621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762505.604660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762506.052587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762506.628750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762507.076670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762507.076697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762507.076711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762507.076717:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762507.076729:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1551 00000100:00080000:3.0:1787762507.076744:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762507.076753:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762507.076804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274216064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762507.076830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762507.076833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762507.076836:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762507.076840:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1551 00000100:00080000:3.0:1787762507.076843:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762507.076847:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762507.076859:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762507.076875:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762507.076875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274216192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762507.076978:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762507.077030:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762507/real 1787762507] req@ffff9ac451c52580 x1874603274216064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762507.077055:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274216064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762507.077070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762507.077075:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762507.077081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762507.077111:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216192:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762507.077138:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762507.077152:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762507/real 1787762507] req@ffff9ac451c50f00 x1874603274216192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762507.077167:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274216192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762507.077178:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762507.077183:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762507.077187:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216192:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762507.652728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762508.100629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762508.676646:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762509.124652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762509.700636:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762510.148676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762510.724728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762511.172658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762511.748787:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762512.196635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762512.196647:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762512.196653:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762512.196661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762512.196666:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1556 00000100:00080000:0.0:1787762512.196684:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762512.196695:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762512.196750:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274216320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762512.196780:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762512.196785:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762512.196787:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762512.196792:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1556 00000100:00080000:0.0:1787762512.196796:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762512.196801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762512.196814:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762512.196828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274216448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762512.196838:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216320:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762512.196919:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762512.196960:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762512/real 1787762512] req@ffff9ac451f87c00 x1874603274216320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762512.196975:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274216320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762512.196987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762512.196992:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762512.196996:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762512.197027:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216448:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762512.197065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762512.197075:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762512/real 1787762512] req@ffff9ac451f84b40 x1874603274216448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762512.197084:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274216448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762512.197090:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762512.197093:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762512.197095:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216448:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762512.772755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762513.220660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762513.796710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762514.244596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762514.820654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762515.268606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762515.844707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762516.292650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762516.868788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762517.316634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762517.316649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762517.316660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762517.316679:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762517.316695:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1562 00000100:00080000:3.0:1787762517.316713:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762517.316722:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762517.316772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274216576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762517.316806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762517.316811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762517.316814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762517.316820:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1562 00000100:00080000:3.0:1787762517.316825:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762517.316832:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762517.316846:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762517.316854:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762517.316861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274216704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762517.316939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762517.316982:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762517/real 1787762517] req@ffff9ac451c50f00 x1874603274216576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762517.317000:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274216576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762517.317014:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762517.317020:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762517.317025:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762517.317059:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216704:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762517.317081:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762517.317101:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762517/real 1787762517] req@ffff9ac451c52580 x1874603274216704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762517.317110:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274216704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762517.317117:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762517.317120:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762517.317122:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216704:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762517.892736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762518.340637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762518.916785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762519.364599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762519.940743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762520.388622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762520.964745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762521.412647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762521.988739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762522.436656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762522.436699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762522.436709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762522.436714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762522.436727:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1567 00000100:00080000:3.0:1787762522.436740:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762522.436750:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762522.436800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274216832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762522.436846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762522.436849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762522.436851:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762522.436854:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1567 00000100:00080000:3.0:1787762522.436857:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762522.436860:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762522.436871:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762522.436882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274216960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762522.436916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216832:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762522.437005:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762522.437059:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762522/real 1787762522] req@ffff9ac451c503c0 x1874603274216832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762522.437079:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274216832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762522.437092:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762522.437097:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762522.437102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274216832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762522.437111:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216960:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762522.437137:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762522.437152:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762522/real 1787762522] req@ffff9ac451c50780 x1874603274216960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762522.437183:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274216960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762522.437192:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762522.437196:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762522.437199:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274216960:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762523.012679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762523.460588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762524.036714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762524.484609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762525.060703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762525.508598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762526.084782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762526.532629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762527.108744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762527.556670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762527.556672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762527.556689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762527.556696:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762527.556711:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1572 00000100:00080000:3.0:1787762527.556729:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762527.556741:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762527.556804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274217088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762527.556847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762527.556852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762527.556854:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762527.556860:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1572 00000100:00080000:3.0:1787762527.556889:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762527.556896:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762527.556912:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762527.556931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274217216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762527.556978:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217088:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762527.557086:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762527.557145:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762527/real 1787762527] req@ffff9ac451c50b40 x1874603274217088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762527.557171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274217088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762527.557186:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762527.557192:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762527.557198:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762527.557206:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217216:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762527.557232:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762527.557246:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762527/real 1787762527] req@ffff9ac451c52940 x1874603274217216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762527.557284:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274217216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762527.557295:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762527.557299:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762527.557303:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217216:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762528.132715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762528.580634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762529.156743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762529.604612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762530.180728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762530.628644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762531.204741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762531.652629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762532.228659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762532.676604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762532.676657:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762532.676663:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762532.676668:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762532.676678:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1577 00000100:00080000:0.0:1787762532.676691:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762532.676698:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762532.676756:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274217344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762532.676800:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762532.676803:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762532.676804:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762532.676808:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1577 00000100:00080000:0.0:1787762532.676811:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762532.676815:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762532.676825:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762532.676835:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274217472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762532.676934:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762532.677063:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762533.252772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762533.700737:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762534.276702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762534.724676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762535.300689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762535.748788:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762535.748938:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762535.749036:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762535.749068:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762535.749149:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762535.749219:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762532/real 1787762535] req@ffff9ac451f84b40 x1874603274217344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762535.749256:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274217344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762535.749292:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762535.749300:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762535.749307:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762535.749319:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762532/real 1787762535] req@ffff9ac451f87c00 x1874603274217472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762535.749339:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274217472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762535.749352:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762535.749383:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762535.749388:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762536.324671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762536.772668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762537.348662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762537.796649:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762537.796663:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762537.796669:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762537.796683:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1582 00000100:00100000:2.0:1787762537.796687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762537.796700:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762537.796731:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762537.796786:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274217600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762537.796829:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762537.796835:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762537.796837:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762537.796843:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1582 00000100:00080000:0.0:1787762537.796847:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762537.796852:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762537.796866:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762537.796880:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274217728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762537.796959:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217600:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762537.797057:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762537.797132:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762537/real 1787762537] req@ffff9ac451f87c00 x1874603274217600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762537.797164:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274217600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762537.797210:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762537.797218:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762537.797226:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762537.797239:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217728:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762537.797315:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762537.797341:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762537/real 1787762537] req@ffff9ac451f84b40 x1874603274217728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762537.797371:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274217728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762537.797387:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762537.797393:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762537.797399:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217728:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762538.372718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762538.820681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762539.396740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762539.844675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762540.420817:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762540.868605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762541.444715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762541.892614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762542.468667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762542.916619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762542.916652:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762542.916658:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762542.916662:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762542.916670:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1587 00000100:00080000:1.0:1787762542.916678:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762542.916684:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762542.916738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274217856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762542.916798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762542.916803:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762542.916805:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762542.916811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1587 00000100:00080000:1.0:1787762542.916816:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762542.916821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762542.916831:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762542.916840:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762542.916848:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274217984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762542.916905:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762542.916959:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762542/real 1787762542] req@ffff9ac447843c00 x1874603274217856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762542.916980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274217856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762542.916993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762542.916999:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762542.917004:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274217856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762542.917029:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217984:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762542.917055:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762542.917069:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762542/real 1787762542] req@ffff9ac447840b40 x1874603274217984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762542.917083:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274217984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762542.917093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762542.917097:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762542.917101:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274217984:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762543.492697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762543.940700:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762544.516677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762544.964632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762545.540784:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762545.988661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762546.564717:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762547.012663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762547.588791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762548.036673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762548.036698:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762548.036710:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762548.036716:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762548.036729:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1592 00000100:00080000:0.0:1787762548.036743:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762548.036767:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762548.036810:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274218112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762548.036851:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762548.036856:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762548.036859:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762548.036864:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1592 00000100:00080000:0.0:1787762548.036870:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762548.036874:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762548.036887:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762548.036906:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274218240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762548.036928:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218112:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762548.037013:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762548.037064:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762548/real 1787762548] req@ffff9ac451f870c0 x1874603274218112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762548.037088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274218112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762548.037101:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762548.037106:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762548.037112:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218112:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762548.037141:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274218240:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762548.037212:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762548.037232:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762548/real 1787762548] req@ffff9ac451f84000 x1874603274218240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762548.037248:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274218240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762548.037260:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762548.037264:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762548.037269:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274218240:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762548.612655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762549.060630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762549.636793:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762550.084645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762550.660733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762551.108640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762551.684661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762552.132600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762552.708742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762553.156583:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762553.156595:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762553.156600:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762553.156613:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1597 00000100:00080000:0.0:1787762553.156629:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762553.156640:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762553.156641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787762553.156700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274218368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762553.156736:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762553.156762:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762553.156765:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762553.156770:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1597 00000100:00080000:0.0:1787762553.156776:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762553.156782:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762553.156796:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762553.156810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762553.156812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274218496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762553.156896:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762553.156950:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762553/real 1787762553] req@ffff9ac451f861c0 x1874603274218368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762553.156969:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274218368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762553.156982:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762553.156987:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762553.156992:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762553.157024:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274218496:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762553.157046:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762553.157056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762553/real 1787762553] req@ffff9ac451f84780 x1874603274218496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762553.157073:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274218496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762553.157080:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762553.157083:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762553.157085:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274218496:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762553.732750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762554.180641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762554.756710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762555.204608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762555.780666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762556.228635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762556.804707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762557.252734:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762557.828694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762558.276634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762558.276635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762558.276653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762558.276659:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762558.276674:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1603 00000100:00080000:0.0:1787762558.276688:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762558.276698:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762558.276743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274218624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762558.276779:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762558.276784:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762558.276787:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762558.276793:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1603 00000100:00080000:0.0:1787762558.276797:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762558.276801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762558.276835:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762558.276854:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762558.276858:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274218752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762558.276948:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762558.277000:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762558/real 1787762558] req@ffff9ac451f85680 x1874603274218624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762558.277024:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274218624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762558.277039:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762558.277044:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762558.277049:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762558.277081:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274218752:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762558.277130:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762558.277150:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762558/real 1787762558] req@ffff9ac451f843c0 x1874603274218752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762558.277189:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274218752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762558.277201:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762558.277205:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762558.277210:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274218752:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762558.852717:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762559.300629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762559.876725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762560.324636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762560.900663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762561.348705:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762561.924699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762562.372604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762562.948635:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762563.396677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762563.396733:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762563.396743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762563.396748:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762563.396759:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1608 00000100:00080000:1.0:1787762563.396772:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762563.396781:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762563.396829:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274218880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762563.396874:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762563.396882:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762563.396885:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762563.396893:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1608 00000100:00080000:1.0:1787762563.396901:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762563.396911:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762563.396928:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762563.396948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762563.396985:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274219008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762563.397056:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762563.397142:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762563/real 1787762563] req@ffff9ac447842940 x1874603274218880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762563.397178:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274218880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762563.397200:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762563.397208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762563.397214:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274218880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762563.397245:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219008:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762563.397278:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762563.397295:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762563/real 1787762563] req@ffff9ac447842d00 x1874603274219008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762563.397311:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274219008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762563.397322:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762563.397326:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762563.397343:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219008:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762563.972640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762564.420629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762564.996664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762565.444613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762566.020702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762566.468611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762567.044651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762567.492619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762568.068707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762568.516613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762568.516683:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762568.516689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762568.516694:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762568.516704:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1613 00000100:00080000:0.0:1787762568.516718:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762568.516726:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762568.516775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274219136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762568.516807:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762568.516810:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762568.516811:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762568.516815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1613 00000100:00080000:0.0:1787762568.516818:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762568.516821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762568.516833:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762568.516845:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274219264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762568.516886:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219136:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762568.516992:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762568.517068:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762568/real 1787762568] req@ffff9ac451f843c0 x1874603274219136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762568.517094:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274219136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762568.517108:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762568.517114:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762568.517119:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762568.517130:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219264:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762568.517159:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762568.517174:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762568/real 1787762568] req@ffff9ac451f85680 x1874603274219264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762568.517189:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274219264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762568.517200:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762568.517204:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762568.517209:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219264:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762569.092687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762569.540603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762570.116768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762570.564609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762571.140731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762571.588598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762572.164716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762572.612675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762573.188677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762573.636656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762573.636685:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762573.636704:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762573.636743:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762573.636759:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1618 00000100:00080000:0.0:1787762573.636779:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762573.636795:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762573.636866:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274219392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762573.636916:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762573.636922:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762573.636925:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762573.636932:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1618 00000100:00080000:0.0:1787762573.636938:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762573.636954:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762573.636969:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762573.636990:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274219520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762573.637021:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219392:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762573.637091:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762573.637131:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762573/real 1787762573] req@ffff9ac451f84780 x1874603274219392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762573.637145:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274219392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762573.637154:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762573.637157:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762573.637160:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762573.637179:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219520:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762573.637195:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762573.637203:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762573/real 1787762573] req@ffff9ac451f861c0 x1874603274219520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762573.637211:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274219520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762573.637217:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762573.637219:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762573.637221:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219520:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762574.212687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762574.660599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762575.236676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762575.684637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762576.260652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762576.708587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762577.284651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762577.732632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762578.308651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762578.756610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762578.756633:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762578.756641:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762578.756645:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762578.756654:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1623 00000100:00080000:0.0:1787762578.756663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762578.756680:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762578.756713:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274219648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762578.756756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762578.756759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762578.756760:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762578.756764:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1623 00000100:00080000:0.0:1787762578.756767:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762578.756770:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762578.756788:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762578.756799:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274219776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762578.756834:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219648:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762578.756937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762578.757045:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762578/real 1787762578] req@ffff9ac451f84000 x1874603274219648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762578.757066:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274219648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762578.757077:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762578.757080:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762578.757084:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762578.757090:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219776:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762578.757110:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762578.757120:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762578/real 1787762578] req@ffff9ac451f870c0 x1874603274219776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762578.757128:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274219776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762578.757133:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762578.757136:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762578.757138:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274219776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762579.332752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762579.780625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762580.356659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762580.804668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762581.380750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762581.828596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762582.404786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762582.852640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762583.428755:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762583.876628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762583.876739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762583.876748:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762583.876770:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762583.876783:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1628 00000100:00080000:1.0:1787762583.876796:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762583.876806:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762583.876849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274219904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762583.876891:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762583.876897:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762583.876899:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762583.876905:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1628 00000100:00080000:1.0:1787762583.876909:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762583.876913:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762583.876925:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762583.876965:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274220032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762583.877071:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219904:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762583.877168:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762583.877218:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762583/real 1787762583] req@ffff9ac4478430c0 x1874603274219904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762583.877241:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274219904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762583.877256:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762583.877261:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762583.877267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274219904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762583.877276:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220032:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762583.877320:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762583.877336:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762583/real 1787762583] req@ffff9ac4478421c0 x1874603274220032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762583.877351:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274220032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762583.877363:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762583.877367:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762583.877371:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762584.452748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762584.900608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762585.476703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762585.924596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762586.500744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762586.948619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762587.524735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762587.972633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762588.548808:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762588.996672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762588.996720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762588.996731:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762588.996737:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762588.996748:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1633 00000100:00080000:1.0:1787762588.996761:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762588.996771:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762588.996820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274220160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762588.996881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762588.996886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762588.996889:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762588.996894:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1633 00000100:00080000:1.0:1787762588.996930:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762588.996937:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762588.996952:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762588.996977:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274220288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762588.996983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220160:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762588.997101:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762588.997156:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762588/real 1787762588] req@ffff9ac4478403c0 x1874603274220160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762588.997174:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274220160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762588.997185:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762588.997189:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762588.997193:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762588.997222:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220288:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762588.997242:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762588.997252:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762588/real 1787762588] req@ffff9ac447841680 x1874603274220288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762588.997278:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274220288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762588.997286:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762588.997289:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762588.997292:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220288:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762589.572665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762590.020584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762590.596667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762591.044634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762591.620790:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762592.068658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762592.644790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762593.092660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762593.668691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762594.116600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762594.116672:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762594.116685:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762594.116690:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762594.116703:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1638 00000100:00080000:0.0:1787762594.116716:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762594.116726:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762594.116775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274220416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762594.116818:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762594.116823:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762594.116826:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762594.116833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1638 00000100:00080000:0.0:1787762594.116837:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762594.116843:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762594.116855:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762594.116877:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762594.116935:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274220544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762594.116982:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762594.117040:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762594/real 1787762594] req@ffff9ac451f84b40 x1874603274220416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762594.117058:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274220416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762594.117068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762594.117072:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762594.117075:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762594.117102:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220544:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762594.117121:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762594.117147:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762594/real 1787762594] req@ffff9ac451f87c00 x1874603274220544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762594.117156:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274220544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762594.117161:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762594.117164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762594.117166:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762594.692711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762595.140609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762595.716649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762596.164639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762596.740695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762597.188652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762597.764719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762598.212706:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762598.788786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762599.236660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762599.236720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762599.236735:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762599.236742:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762599.236758:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1643 00000100:00080000:0.0:1787762599.236773:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762599.236786:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762599.236869:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274220672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762599.236908:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762599.236913:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762599.236916:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762599.236922:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1643 00000100:00080000:0.0:1787762599.236928:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762599.236934:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762599.236947:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762599.236968:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274220800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762599.237003:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762599.237170:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762599.812779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762600.260769:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762600.836718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762601.284721:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762601.860761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762602.308892:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762602.309061:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762602.309138:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762602.309163:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762602.309232:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762602.309347:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762599/real 1787762602] req@ffff9ac451f87c00 x1874603274220672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762602.309413:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274220672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762602.309429:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762602.309435:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762602.309440:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762602.309450:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762599/real 1787762602] req@ffff9ac451f84b40 x1874603274220800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762602.309466:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274220800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762602.309475:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762602.309480:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762602.309484:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274220800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762602.884773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762603.332682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762603.908719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762604.356685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762604.356697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762604.356702:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787762604.356704:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762604.356716:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1649 00000100:00080000:3.0:1787762604.356730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762604.356739:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762604.356813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274220928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762604.356840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762604.356845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762604.356847:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762604.356852:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1649 00000100:00080000:3.0:1787762604.356856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762604.356860:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762604.356891:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762604.356916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274221056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762604.356934:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220928:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762604.357079:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762604.357148:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762604/real 1787762604] req@ffff9ac451c530c0 x1874603274220928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762604.357186:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274220928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762604.357207:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762604.357215:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762604.357245:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274220928:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762604.357293:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221056:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762604.357347:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762604.357375:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762604/real 1787762604] req@ffff9ac451c52d00 x1874603274221056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762604.357404:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274221056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762604.357423:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762604.357430:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762604.357437:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221056:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762604.932725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762605.380771:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762605.956779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762606.404649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762606.980837:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762607.428672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762608.004816:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762608.452676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762609.028728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762609.476664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762609.476677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762609.476685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762609.476685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762609.476702:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1654 00000100:00080000:3.0:1787762609.476717:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762609.476729:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762609.476794:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274221184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762609.476820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762609.476823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762609.476825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762609.476828:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1654 00000100:00080000:3.0:1787762609.476831:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762609.476834:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762609.476844:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762609.476855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274221312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762609.476871:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221184:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762609.476950:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762609.476994:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762609/real 1787762609] req@ffff9ac451c52d00 x1874603274221184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762609.477009:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274221184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762609.477019:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762609.477023:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762609.477026:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762609.477055:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221312:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762609.477091:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762609.477101:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762609/real 1787762609] req@ffff9ac451c530c0 x1874603274221312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762609.477110:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274221312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762609.477116:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762609.477118:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762609.477120:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762610.052721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762610.500678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762611.076773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762611.524649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762612.100842:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762612.548662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762613.124816:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762613.572655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762614.148756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762614.596631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762614.596643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762614.596649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762614.596665:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1659 00000100:00100000:2.0:1787762614.596667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762614.596706:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762614.596716:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762614.596777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274221440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762614.596805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762614.596808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762614.596809:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762614.596813:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1659 00000100:00080000:3.0:1787762614.596815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762614.596818:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762614.596829:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762614.596838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274221568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762614.596859:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221440:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762614.596941:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762614.596987:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762614/real 1787762614] req@ffff9ac451c52940 x1874603274221440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762614.597002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274221440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762614.597012:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762614.597015:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762614.597018:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762614.597044:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221568:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762614.597059:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762614.597078:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762614/real 1787762614] req@ffff9ac451c50b40 x1874603274221568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762614.597087:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274221568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762614.597093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762614.597096:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762614.597098:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221568:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762615.172786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762615.620671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762616.196740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762616.644628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762617.220688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762617.668620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762618.244745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762618.692664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762619.268847:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762619.716630:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762619.716641:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762619.716647:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762619.716660:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1664 00000100:00080000:0.0:1787762619.716674:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762619.716682:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762619.716722:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787762619.716749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274221696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762619.716782:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762619.716786:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762619.716787:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762619.716791:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1664 00000100:00080000:0.0:1787762619.716794:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762619.716798:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762619.716810:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762619.716837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274221824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762619.716844:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221696:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762619.716937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762619.716990:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762619/real 1787762619] req@ffff9ac451f84b40 x1874603274221696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762619.717007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274221696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762619.717017:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762619.717021:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762619.717024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221696:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762619.717050:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221824:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762619.717069:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762619.717078:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762619/real 1787762619] req@ffff9ac451f87c00 x1874603274221824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762619.717087:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274221824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762619.717093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762619.717096:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762619.717098:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274221824:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762620.292704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762620.740655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762621.316796:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762621.764663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762622.340741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762622.788624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762623.364799:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762623.812636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762624.388771:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762624.836632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762624.836717:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762624.836727:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762624.836759:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762624.836771:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1669 00000100:00080000:1.0:1787762624.836785:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762624.836793:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762624.836837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274221952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762624.836872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762624.836875:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762624.836877:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762624.836881:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1669 00000100:00080000:1.0:1787762624.836883:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762624.836886:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762624.836898:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762624.836910:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274222080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762624.836951:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221952:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762624.837048:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762624.837120:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762624/real 1787762624] req@ffff9ac447840780 x1874603274221952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762624.837158:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274221952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762624.837174:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762624.837179:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762624.837184:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274221952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762624.837195:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222080:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762624.837229:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762624.837247:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762624/real 1787762624] req@ffff9ac447840f00 x1874603274222080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762624.837267:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274222080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762624.837285:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762624.837292:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762624.837299:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222080:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762625.412703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762625.860615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762626.436781:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762626.884618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762627.460776:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762627.908660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762628.484711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762628.932688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762629.508818:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762629.956694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762629.956702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762629.956717:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762629.956724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762629.956740:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1674 00000100:00080000:0.0:1787762629.956757:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762629.956768:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762629.956819:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274222208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762629.956862:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762629.956867:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762629.956870:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762629.956875:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1674 00000100:00080000:0.0:1787762629.956881:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762629.956887:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762629.956901:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762629.956924:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762629.956925:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274222336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762629.957019:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762629.957071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762629/real 1787762629] req@ffff9ac451f870c0 x1874603274222208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762629.957094:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274222208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762629.957108:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762629.957113:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762629.957135:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762629.957166:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222336:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762629.957198:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762629.957213:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762629/real 1787762629] req@ffff9ac451f84000 x1874603274222336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762629.957229:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274222336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762629.957239:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762629.957244:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762629.957248:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222336:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762630.532756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762630.980679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762631.556791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762632.004688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762632.580800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762633.028670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762633.604719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762634.052661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762634.628726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762635.076632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762635.076764:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762635.076774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762635.076780:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762635.076792:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1679 00000100:00080000:1.0:1787762635.076805:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762635.076814:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762635.076861:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274222464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762635.076902:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762635.076908:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762635.076910:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762635.076915:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1679 00000100:00080000:1.0:1787762635.076920:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762635.076947:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762635.076960:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762635.076971:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762635.076987:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274222592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762635.077061:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762635.077127:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762635/real 1787762635] req@ffff9ac447840000 x1874603274222464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762635.077151:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274222464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762635.077166:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762635.077171:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762635.077177:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762635.077204:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222592:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762635.077232:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762635.077247:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762635/real 1787762635] req@ffff9ac447842580 x1874603274222592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762635.077262:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274222592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762635.077273:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762635.077277:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762635.077282:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222592:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762635.652782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762636.100647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762636.676757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762637.124688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762637.700765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762638.148708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762638.724758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762639.172691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762639.748746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762640.196675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762640.196687:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762640.196697:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762640.196703:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762640.196715:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1684 00000100:00080000:1.0:1787762640.196729:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762640.196739:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762640.196786:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274222720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762640.196823:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762640.196828:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762640.196831:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762640.196836:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1684 00000100:00080000:1.0:1787762640.196840:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762640.196844:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762640.196856:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762640.196872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274222848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762640.196925:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222720:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762640.197009:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762640.197061:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762640/real 1787762640] req@ffff9ac447843840 x1874603274222720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762640.197114:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274222720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762640.197127:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762640.197133:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762640.197138:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762640.197148:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222848:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762640.197178:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762640.197193:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762640/real 1787762640] req@ffff9ac447841a40 x1874603274222848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762640.197208:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274222848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762640.197219:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762640.197223:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762640.197227:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274222848:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762640.772753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762641.220670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762641.796765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762642.244663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762642.820783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762643.268681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762643.844755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762644.292686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762644.868695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762645.316685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762645.316733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762645.316743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762645.316749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762645.316784:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1690 00000100:00080000:2.0:1787762645.316802:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762645.316814:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762645.316862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274222976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762645.316902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762645.316907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762645.316910:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762645.316915:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1690 00000100:00080000:2.0:1787762645.316920:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762645.316924:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762645.316938:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762645.316956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274223104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762645.316970:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222976:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762645.317056:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762645.317107:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762645/real 1787762645] req@ffff9ac4523bf0c0 x1874603274222976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762645.317153:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274222976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762645.317167:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762645.317173:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762645.317178:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274222976:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762645.317211:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223104:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762645.317256:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762645.317283:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762645/real 1787762645] req@ffff9ac4523be1c0 x1874603274223104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762645.317311:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274223104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762645.317326:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762645.317330:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762645.317335:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223104:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762645.892757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762646.340646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762646.916701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762647.364674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762647.940707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762648.388637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762648.964715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762649.412629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762649.988727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762650.436674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762650.436677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762650.436711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762650.436718:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762650.436732:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1695 00000100:00080000:3.0:1787762650.436747:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762650.436757:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762650.436808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274223232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762650.436851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762650.436855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762650.436857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762650.436862:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1695 00000100:00080000:3.0:1787762650.436866:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762650.436870:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762650.436884:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762650.436899:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274223360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762650.436909:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274223232:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762650.436992:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762650.437034:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762650/real 1787762650] req@ffff9ac451c50780 x1874603274223232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762650.437049:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274223232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762650.437066:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762650.437070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762650.437073:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274223232:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762650.437099:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223360:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762650.437118:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762650.437127:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762650/real 1787762650] req@ffff9ac451c503c0 x1874603274223360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762650.437136:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274223360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762650.437142:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762650.437144:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762650.437147:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223360:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762651.012731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762651.460660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762652.036667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762652.484603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762653.060778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762653.508627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762654.084690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762654.532603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762655.108702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762655.556598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762655.556634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762655.556642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762655.556647:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762655.556657:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1700 00000100:00080000:3.0:1787762655.556670:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762655.556678:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762655.556730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274223488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762655.556773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762655.556776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762655.556778:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762655.556782:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1700 00000100:00080000:3.0:1787762655.556786:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762655.556790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762655.556800:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762655.556804:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274223488:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762655.556814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274223616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762655.556877:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762655.556919:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762655/real 1787762655] req@ffff9ac451c52580 x1874603274223488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762655.556933:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274223488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762655.556954:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762655.556958:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762655.556961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274223488:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762655.556985:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223616:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762655.557003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762655.557012:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762655/real 1787762655] req@ffff9ac451c50f00 x1874603274223616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762655.557021:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274223616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762655.557027:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762655.557029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762655.557031:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223616:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762656.132674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762656.580698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762657.156712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762657.604661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762658.180718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762658.628624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762659.204683:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762659.652626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762660.228770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762660.676641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762660.676643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762660.676655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762660.676663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762660.676681:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1705 00000100:00080000:3.0:1787762660.676700:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762660.676715:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762660.676780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274223744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762660.676821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762660.676826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762660.676829:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762660.676835:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1705 00000100:00080000:3.0:1787762660.676862:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762660.676868:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762660.676882:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762660.676902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274223872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762660.676945:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274223744:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762660.677065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762660.677124:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762660/real 1787762660] req@ffff9ac451c50f00 x1874603274223744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762660.677149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274223744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762660.677165:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762660.677195:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762660.677201:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274223744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762660.677212:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223872:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762660.677246:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762660.677263:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762660/real 1787762660] req@ffff9ac451c52580 x1874603274223872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762660.677279:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274223872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762660.677290:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762660.677294:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762660.677298:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274223872:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762661.252780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762661.700661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762662.276793:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762662.724632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762663.300713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762663.748662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762664.324734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762664.772664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762665.348686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762665.796611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762665.796615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762665.796627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762665.796633:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762665.796645:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1710 00000100:00080000:3.0:1787762665.796656:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762665.796665:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762665.796723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274224000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762665.796765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762665.796771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762665.796773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762665.796779:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1710 00000100:00080000:3.0:1787762665.796792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762665.796798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762665.796811:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762665.796845:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274224128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762665.796865:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224000:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762665.796964:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224128:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762666.372763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762666.820593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762667.396754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762667.844653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762668.420775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762668.868655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787762668.868942:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787762668.869014:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787762668.869040:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787762668.869111:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762668.869177:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762665/real 1787762668] req@ffff9ac451c50780 x1874603274224128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762668.869215:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274224128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762668.869229:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762668.869256:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762668.869261:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224128:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787762668.869270:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762665/real 1787762668] req@ffff9ac451c503c0 x1874603274224000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762668.869286:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274224000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762668.869294:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762668.869297:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762668.869301:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224000:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787762669.444747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762669.892649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762670.468784:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762670.916636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762670.916665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762670.916679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762670.916685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762670.916700:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1715 00000100:00080000:3.0:1787762670.916715:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762670.916727:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762670.916774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274224256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762670.916807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762670.916812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762670.916814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762670.916820:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1715 00000100:00080000:3.0:1787762670.916824:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762670.916830:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762670.916843:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762670.916863:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274224384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762670.916869:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224256:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762670.916987:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762670.917051:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762670/real 1787762670] req@ffff9ac451c50b40 x1874603274224256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762670.917078:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274224256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762670.917103:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762670.917112:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762670.917120:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224256:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762670.917639:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224384:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762670.917697:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762670.917727:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762670/real 1787762670] req@ffff9ac451c52940 x1874603274224384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762670.917778:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274224384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762670.917796:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762670.917803:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762670.917810:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224384:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762671.492654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762671.940594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762672.516726:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762672.964659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762673.541202:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762673.988701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762674.564779:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762675.012657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762675.588730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762676.036666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762676.036702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762676.036715:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762676.036742:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762676.036758:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1720 00000100:00080000:1.0:1787762676.036775:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762676.036785:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762676.036837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274224512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762676.036892:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762676.036900:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762676.036904:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762676.036913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1720 00000100:00080000:1.0:1787762676.036921:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762676.036929:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762676.036946:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762676.036972:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274224640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762676.036976:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224512:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762676.037093:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762676.037151:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762676/real 1787762676] req@ffff9ac447841e00 x1874603274224512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762676.037175:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274224512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762676.037189:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762676.037195:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762676.037200:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224512:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762676.037408:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224640:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762676.037477:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762676.037503:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762676/real 1787762676] req@ffff9ac4478412c0 x1874603274224640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762676.037569:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274224640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762676.037579:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762676.037582:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762676.037585:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224640:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762676.612750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762677.060662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762677.636770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762678.084646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762678.660732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762679.108601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762679.684659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762680.132646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762680.708720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762681.156641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762681.156692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762681.156702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762681.156708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762681.156719:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1725 00000100:00080000:0.0:1787762681.156732:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762681.156741:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762681.156786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274224768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762681.156833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762681.156838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762681.156841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762681.156846:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1725 00000100:00080000:0.0:1787762681.156850:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762681.156856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762681.156870:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762681.156906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274224896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762681.156914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224768:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762681.157009:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762681.157057:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762681/real 1787762681] req@ffff9ac451f861c0 x1874603274224768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762681.157081:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274224768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762681.157115:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762681.157121:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762681.157126:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274224768:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762681.157156:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224896:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762681.157189:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762681.157205:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762681/real 1787762681] req@ffff9ac451f84780 x1874603274224896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762681.157220:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274224896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762681.157231:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762681.157235:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762681.157239:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274224896:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762681.732728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762682.180647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762682.756756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762683.204658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762683.780824:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762684.228671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762684.804782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762685.252621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762685.828793:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762686.276615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762686.276647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762686.276656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762686.276673:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762686.276684:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1731 00000100:00080000:3.0:1787762686.276695:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762686.276702:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762686.276743:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274225024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762686.276782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762686.276785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762686.276786:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762686.276789:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1731 00000100:00080000:3.0:1787762686.276791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762686.276794:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762686.276804:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762686.276813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274225152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762686.276840:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225024:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762686.276926:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762686.276976:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762686/real 1787762686] req@ffff9ac451c530c0 x1874603274225024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762686.276992:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274225024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762686.277002:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762686.277014:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762686.277020:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225024:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762686.277028:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225152:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762686.277055:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762686.277067:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762686/real 1787762686] req@ffff9ac451c52d00 x1874603274225152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762686.277076:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274225152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762686.277083:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762686.277086:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762686.277088:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225152:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762686.852746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762687.300654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762687.876754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762688.324662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762688.900768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762689.348657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762689.924742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762690.372649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762690.948750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762691.396646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762691.396655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762691.396661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762691.396670:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762691.396683:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1736 00000100:00080000:3.0:1787762691.396697:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762691.396707:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762691.396770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274225280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762691.396808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762691.396832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762691.396835:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762691.396841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1736 00000100:00080000:3.0:1787762691.396847:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762691.396853:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762691.396864:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762691.396890:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274225408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762691.396959:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225280:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762691.397048:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762691.397121:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762691/real 1787762691] req@ffff9ac451c52d00 x1874603274225280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762691.397160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274225280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762691.397183:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762691.397191:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762691.397220:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225280:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762691.397246:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225408:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762691.397282:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762691.397301:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762691/real 1787762691] req@ffff9ac451c530c0 x1874603274225408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762691.397318:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274225408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762691.397329:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762691.397333:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762691.397337:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225408:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762691.972754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762692.420634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762692.996747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762693.444656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762694.020643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762694.468598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762695.044784:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762695.492667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762696.068784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762696.516653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762696.516681:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762696.516690:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762696.516694:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762696.516703:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1741 00000100:00080000:1.0:1787762696.516713:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762696.516718:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762696.516752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274225536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762696.516781:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762696.516783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762696.516785:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762696.516788:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1741 00000100:00080000:1.0:1787762696.516806:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762696.516810:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762696.516823:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762696.516827:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225536:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762696.516837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274225664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762696.516885:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762696.516921:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762696/real 1787762696] req@ffff9ac447843480 x1874603274225536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762696.516938:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274225536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762696.516950:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762696.516954:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762696.516959:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225536:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762696.516983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225664:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762696.517002:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762696.517011:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762696/real 1787762696] req@ffff9ac4478412c0 x1874603274225664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762696.517020:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274225664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762696.517032:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762696.517035:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762696.517038:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225664:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762697.092782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762697.540640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762698.116782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762698.564635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762699.140810:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762699.588676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762700.164728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762700.612675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762701.188628:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762701.636577:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762701.636617:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762701.636622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762701.636626:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762701.636634:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1746 00000100:00080000:0.0:1787762701.636643:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762701.636648:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762701.636683:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274225792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762701.636715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762701.636720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762701.636722:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762701.636727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1746 00000100:00080000:0.0:1787762701.636731:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762701.636735:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762701.636744:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225792:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787762701.636746:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762701.636768:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274225920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762701.636805:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762701.636839:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762701/real 1787762701] req@ffff9ac451f85680 x1874603274225792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762701.636852:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274225792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762701.636860:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762701.636863:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762701.636866:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274225792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762701.636884:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225920:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762701.636908:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762701.636916:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762701/real 1787762701] req@ffff9ac451f843c0 x1874603274225920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762701.636924:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274225920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762701.636947:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762701.636950:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762701.636952:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274225920:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762702.212723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762702.660575:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762703.236636:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762703.684598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762704.260785:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762704.708606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762705.284649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762705.732610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762706.308704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762706.756628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762706.756641:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762706.756654:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762706.756661:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762706.756675:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1751 00000100:00080000:0.0:1787762706.756694:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762706.756709:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762706.756776:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274226048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762706.756821:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762706.756826:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762706.756840:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762706.756847:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1751 00000100:00080000:0.0:1787762706.756853:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762706.756859:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762706.756873:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762706.756893:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274226176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762706.756948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226048:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762706.757048:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762706.757140:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762706/real 1787762706] req@ffff9ac451f843c0 x1874603274226048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762706.757167:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274226048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762706.757183:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762706.757188:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762706.757194:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226048:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762706.757204:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226176:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762706.757237:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762706.757253:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762706/real 1787762706] req@ffff9ac451f85680 x1874603274226176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762706.757269:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274226176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762706.757279:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762706.757307:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762706.757312:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226176:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762707.332681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762707.780603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762708.356695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762708.804597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762709.380685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762709.828636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762710.404753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762710.852621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762711.428678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762711.876635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762711.876644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762711.876646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762711.876655:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762711.876667:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1756 00000100:00080000:0.0:1787762711.876697:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762711.876706:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762711.876743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274226304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762711.876774:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762711.876778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762711.876780:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762711.876784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1756 00000100:00080000:0.0:1787762711.876787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762711.876790:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762711.876801:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762711.876814:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274226432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762711.876875:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226304:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762711.877015:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762711.877125:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762711/real 1787762711] req@ffff9ac451f84780 x1874603274226304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762711.877164:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274226304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762711.877181:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762711.877187:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762711.877193:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226304:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762711.877205:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226432:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762711.877242:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762711.877259:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762711/real 1787762711] req@ffff9ac451f861c0 x1874603274226432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762711.877275:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274226432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762711.877286:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762711.877291:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762711.877295:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226432:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762712.452673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762712.900592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762713.476685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762713.924588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762714.500666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762714.948626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762715.524694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762715.972592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762716.548708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762716.996633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762716.996646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762716.996664:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762716.996668:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762716.996676:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1761 00000100:00080000:0.0:1787762716.996686:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762716.996693:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762716.996724:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274226560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762716.996746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762716.996760:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762716.996761:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762716.996765:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1761 00000100:00080000:0.0:1787762716.996768:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762716.996771:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762716.996780:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762716.996789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274226688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762716.996822:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226560:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762716.996878:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762716.996911:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762716/real 1787762716] req@ffff9ac451f84000 x1874603274226560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762716.996923:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274226560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762716.996931:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762716.996934:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762716.996937:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226560:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762716.996942:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226688:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762716.996955:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762716.996963:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762716/real 1787762716] req@ffff9ac451f870c0 x1874603274226688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762716.996970:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274226688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762716.996975:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762716.996977:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762716.996979:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226688:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762717.572655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762718.020620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762718.596799:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762719.044658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762719.620727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762720.068627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762720.644660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762721.092610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762721.668640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762722.116615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762722.116628:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762722.116637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762722.116642:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762722.116653:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1766 00000100:00080000:0.0:1787762722.116664:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762722.116673:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762722.116714:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274226816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762722.116740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762722.116745:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762722.116747:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762722.116751:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1766 00000100:00080000:0.0:1787762722.116756:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762722.116760:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762722.116785:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226816:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787762722.116787:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762722.116808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274226944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762722.116844:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762722.116879:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762722/real 1787762722] req@ffff9ac451f87c00 x1874603274226816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762722.116892:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274226816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762722.116901:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762722.116904:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762722.116907:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274226816:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762722.116925:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226944:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762722.116939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762722.116947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762722/real 1787762722] req@ffff9ac451f84b40 x1874603274226944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762722.116955:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274226944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762722.116961:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762722.116964:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762722.116966:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274226944:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762722.692671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762723.140660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762723.716783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762724.164649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762724.740747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762725.188650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762725.764658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762726.212586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762726.788661:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762727.236601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762727.236648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762727.236653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762727.236657:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762727.236665:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1771 00000100:00080000:3.0:1787762727.236674:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762727.236680:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762727.236711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274227072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762727.236733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762727.236736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762727.236737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762727.236740:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1771 00000100:00080000:3.0:1787762727.236742:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762727.236744:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762727.236752:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762727.236761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274227200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762727.236775:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227072:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762727.236846:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762727.236880:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762727/real 1787762727] req@ffff9ac451c52940 x1874603274227072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762727.236893:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274227072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762727.236901:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762727.236904:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762727.236907:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227072:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762727.236926:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227200:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762727.236940:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762727.236964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762727/real 1787762727] req@ffff9ac451c50b40 x1874603274227200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762727.236973:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274227200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762727.236978:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762727.236981:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762727.236983:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227200:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762727.812675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762728.260597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762728.836648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762729.284597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762729.860699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762730.308576:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762730.884750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762731.332586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762731.908672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762732.356643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762732.356653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762732.356659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762732.356669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762732.356697:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1777 00000100:00080000:3.0:1787762732.356715:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762732.356726:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762732.356782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274227328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762732.356839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762732.356844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762732.356846:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762732.356852:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1777 00000100:00080000:3.0:1787762732.356856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762732.356862:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762732.356892:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762732.356909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274227456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762732.356947:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227328:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762732.357071:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227456:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762732.932674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762733.380732:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762733.956748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762734.404758:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762734.980707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762735.428837:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:3.0:1787762735.429010:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:3.0:1787762735.429085:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787762735.429106:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:3.0:1787762735.429160:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787762735.429243:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762732/real 1787762735] req@ffff9ac451c50780 x1874603274227456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762735.429291:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274227456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762735.429308:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762735.429315:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762735.429320:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227456:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787762735.429332:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762732/real 1787762735] req@ffff9ac451c503c0 x1874603274227328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762735.429349:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274227328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762735.429357:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762735.429361:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762735.429365:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227328:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787762736.004663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762736.452719:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762737.028657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762737.476619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762737.476631:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762737.476635:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762737.476648:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1782 00000100:00080000:1.0:1787762737.476664:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762737.476674:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762737.476679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787762737.476736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274227584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762737.476789:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762737.476793:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762737.476795:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762737.476800:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1782 00000100:00080000:1.0:1787762737.476805:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762737.476810:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762737.476824:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762737.476840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274227712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762737.476853:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227584:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762737.476962:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762737.477015:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762737/real 1787762737] req@ffff9ac447841e00 x1874603274227584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762737.477038:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274227584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762737.477051:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762737.477056:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762737.477061:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762737.477091:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227712:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762737.477119:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762737.477135:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762737/real 1787762737] req@ffff9ac447841a40 x1874603274227712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762737.477166:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274227712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762737.477175:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762737.477179:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762737.477182:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227712:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762738.052657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762738.500806:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762739.076729:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762739.524642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762740.100814:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762740.548734:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762741.124764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762741.572684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762742.148717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762742.596618:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762742.596622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762742.596629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762742.596634:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762742.596642:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1787 00000100:00080000:1.0:1787762742.596652:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762742.596659:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762742.596707:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274227840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762742.596737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762742.596742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762742.596744:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762742.596749:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1787 00000100:00080000:1.0:1787762742.596752:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762742.596756:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762742.596778:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762742.596792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274227968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762742.596814:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227840:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762742.596909:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762742.596957:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762742/real 1787762742] req@ffff9ac447843840 x1874603274227840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762742.596971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274227840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762742.596981:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762742.596984:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762742.596988:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274227840:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762742.596994:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227968:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762742.597010:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762742.597019:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762742/real 1787762742] req@ffff9ac447842580 x1874603274227968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762742.597037:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274227968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762742.597052:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762742.597055:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762742.597058:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274227968:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762743.172746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762743.620601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762744.196777:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762744.644579:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762745.220640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762745.668593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762746.244788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762746.692582:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762747.268728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762747.716654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762747.716688:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762747.716700:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762747.716706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762747.716718:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1792 00000100:00080000:1.0:1787762747.716733:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762747.716743:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762747.716795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274228096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762747.716857:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762747.716862:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762747.716865:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762747.716871:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1792 00000100:00080000:1.0:1787762747.716876:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762747.716880:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762747.716893:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762747.716910:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274228224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762747.716963:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228096:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762747.717065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762747.717127:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762747/real 1787762747] req@ffff9ac447840000 x1874603274228096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762747.717151:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274228096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762747.717166:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762747.717172:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762747.717177:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228096:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762747.717187:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228224:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762747.717219:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762747.717235:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762747/real 1787762747] req@ffff9ac447840f00 x1874603274228224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762747.717251:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274228224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762747.717296:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762747.717304:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762747.717311:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228224:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762748.292792:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762748.740647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762749.316746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762749.764614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762750.340669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762750.788725:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762751.364770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762751.812672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762752.388658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762752.836598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762752.836689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762752.836695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762752.836699:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762752.836709:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1797 00000100:00080000:3.0:1787762752.836723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762752.836732:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762752.836794:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274228352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762752.836830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762752.836833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762752.836835:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762752.836843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1797 00000100:00080000:3.0:1787762752.836846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762752.836850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762752.836861:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762752.836874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274228480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762752.836882:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228352:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762752.836968:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762752.837022:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762752/real 1787762752] req@ffff9ac451c52580 x1874603274228352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762752.837038:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274228352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762752.837048:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762752.837052:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762752.837055:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228352:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762752.837081:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228480:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762752.837101:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762752.837111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762752/real 1787762752] req@ffff9ac451c50f00 x1874603274228480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762752.837119:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274228480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762752.837125:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762752.837143:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762752.837145:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228480:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762753.412721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762753.860659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762754.436649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762754.884598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762755.460660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762755.908621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762756.484812:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762756.932636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762757.508693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762757.956608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762757.956614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762757.956630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762757.956634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762757.956642:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1802 00000100:00080000:3.0:1787762757.956652:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762757.956658:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762757.956689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274228608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762757.956712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762757.956715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762757.956716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762757.956719:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1802 00000100:00080000:3.0:1787762757.956721:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762757.956723:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762757.956731:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762757.956758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274228736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762757.956769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228608:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762757.956825:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762757.956861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762757/real 1787762757] req@ffff9ac451c50f00 x1874603274228608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762757.956874:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274228608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762757.956882:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762757.956885:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762757.956888:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762757.956907:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228736:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762757.956925:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762757.956937:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762757/real 1787762757] req@ffff9ac451c52580 x1874603274228736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762757.956950:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274228736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762757.956959:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762757.956963:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762757.956967:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228736:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762758.532805:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762758.980660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762759.556759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762760.004645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762760.580749:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762761.028631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762761.604704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762762.052674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762762.628781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762763.076647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762763.076770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762763.076780:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762763.076786:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762763.076800:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1807 00000100:00080000:1.0:1787762763.076849:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762763.076860:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762763.076915:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274228864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762763.076954:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762763.076958:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762763.076961:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762763.076966:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1807 00000100:00080000:1.0:1787762763.076970:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762763.076974:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762763.076988:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762763.077002:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274228992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762763.077025:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228864:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762763.077124:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762763.077180:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762763/real 1787762763] req@ffff9ac447840780 x1874603274228864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762763.077203:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274228864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762763.077217:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762763.077223:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762763.077228:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274228864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762763.077277:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228992:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762763.077304:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762763.077319:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762763/real 1787762763] req@ffff9ac447841680 x1874603274228992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762763.077332:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274228992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762763.077342:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762763.077346:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762763.077350:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274228992:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762763.652694:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762764.100624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762764.676675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762765.124653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762765.700674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762766.148590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762766.724790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762767.172645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762767.748770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762768.196603:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762768.196614:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762768.196620:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762768.196631:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1812 00000100:00080000:1.0:1787762768.196643:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762768.196653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762768.196655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787762768.196704:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274229120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762768.196727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762768.196729:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762768.196748:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762768.196752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1812 00000100:00080000:1.0:1787762768.196756:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762768.196760:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762768.196769:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762768.196776:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762768.196782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274229248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762768.196833:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762768.196876:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762768/real 1787762768] req@ffff9ac4478403c0 x1874603274229120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762768.196889:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274229120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762768.196898:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762768.196901:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762768.196904:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762768.196922:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274229248:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762768.196937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762768.196946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762768/real 1787762768] req@ffff9ac4478421c0 x1874603274229248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762768.196955:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274229248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762768.196961:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762768.196963:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762768.196966:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274229248:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762768.772657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762769.220595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762769.796710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762770.244629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762770.820660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762771.268614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762771.844698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762772.292605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762772.868711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762773.316613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762773.316695:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762773.316702:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762773.316731:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762773.316741:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1818 00000100:00080000:0.0:1787762773.316755:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762773.316763:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762773.316807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274229376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762773.316840:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762773.316842:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762773.316844:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762773.316847:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1818 00000100:00080000:0.0:1787762773.316849:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762773.316852:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762773.316863:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762773.316873:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274229504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762773.316922:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229376:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762773.317022:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762773.317076:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762773/real 1787762773] req@ffff9ac451f84b40 x1874603274229376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762773.317101:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274229376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762773.317117:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762773.317122:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762773.317128:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229376:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762773.317138:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274229504:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762773.317171:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762773.317193:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762773/real 1787762773] req@ffff9ac451f87c00 x1874603274229504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762773.317219:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274229504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762773.317275:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762773.317282:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762773.317287:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274229504:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762773.892749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762774.340656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762774.916690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762775.364634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762775.940709:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762776.388665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762776.964706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762777.412623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762777.988690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762778.436636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762778.436651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762778.436662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762778.436669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762778.436687:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1823 00000100:00080000:3.0:1787762778.436703:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762778.436713:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762778.436774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274229632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762778.436817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762778.436821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762778.436824:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762778.436828:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1823 00000100:00080000:3.0:1787762778.436832:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762778.436837:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762778.436850:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762778.436868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274229760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762778.436905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229632:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762778.437018:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762778.437075:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762778/real 1787762778] req@ffff9ac451c503c0 x1874603274229632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762778.437100:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274229632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762778.437117:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762778.437122:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762778.437128:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229632:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762778.437161:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274229760:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762778.437189:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762778.437204:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762778/real 1787762778] req@ffff9ac451c50780 x1874603274229760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762778.437229:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274229760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762778.437245:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762778.437252:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762778.437260:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274229760:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762779.012760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762779.460685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762780.036772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762780.484676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762781.060737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762781.508691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762782.084744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762782.532741:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762783.108781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762783.556682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762783.556717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762783.556727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762783.556734:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762783.556748:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1828 00000100:00080000:3.0:1787762783.556764:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762783.556794:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762783.556846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274229888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762783.556886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762783.556892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762783.556894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762783.556900:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1828 00000100:00080000:3.0:1787762783.556906:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762783.556911:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762783.556925:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762783.556946:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762783.556960:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274230016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762783.557018:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762783.557067:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762783/real 1787762783] req@ffff9ac451c50b40 x1874603274229888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762783.557082:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274229888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762783.557091:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762783.557094:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762783.557097:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274229888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762783.557119:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230016:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762783.557136:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762783.557144:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762783/real 1787762783] req@ffff9ac451c52940 x1874603274230016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762783.557153:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274230016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762783.557159:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762783.557161:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762783.557164:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230016:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762784.132747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762784.580688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762785.156773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762785.604690:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762786.180761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762786.628610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762787.204722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762787.652608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762788.228738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762788.676632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762788.676643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762788.676643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762788.676649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762788.676662:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1833 00000100:00080000:3.0:1787762788.676676:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762788.676684:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762788.676731:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274230144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762788.676758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762788.676761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762788.676785:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762788.676793:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1833 00000100:00080000:3.0:1787762788.676798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762788.676805:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762788.676818:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762788.676834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274230272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762788.676857:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230144:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762788.676960:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762788.677018:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762788/real 1787762788] req@ffff9ac451c530c0 x1874603274230144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762788.677063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274230144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762788.677078:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762788.677084:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762788.677090:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230144:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762788.677126:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230272:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762788.677158:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762788.677173:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762788/real 1787762788] req@ffff9ac451c52d00 x1874603274230272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762788.677189:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274230272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762788.677200:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762788.677205:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762788.677209:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230272:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762789.252785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762789.700680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762790.276784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762790.724641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762791.300669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762791.748627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762792.324699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762792.772652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762793.348759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762793.796599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762793.796639:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762793.796647:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762793.796652:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762793.796663:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1838 00000100:00080000:1.0:1787762793.796675:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762793.796683:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762793.796728:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274230400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762793.796779:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762793.796783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762793.796784:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762793.796789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1838 00000100:00080000:1.0:1787762793.796793:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762793.796799:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762793.796810:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762793.796819:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230400:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762793.796826:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274230528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762793.796902:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762793.796940:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762793/real 1787762793] req@ffff9ac4478430c0 x1874603274230400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762793.796954:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274230400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762793.796963:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762793.796966:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762793.796969:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230400:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762793.796988:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230528:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762793.797003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762793.797011:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762793/real 1787762793] req@ffff9ac447842d00 x1874603274230528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762793.797019:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274230528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762793.797042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762793.797045:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762793.797047:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230528:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762794.372662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762794.820602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762795.396744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762795.844629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762796.420728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762796.868618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762797.444663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762797.892666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762798.468714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762798.916600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762798.916600:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762798.916608:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762798.916612:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762798.916620:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1843 00000100:00080000:1.0:1787762798.916630:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762798.916637:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762798.916678:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274230656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762798.916708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762798.916712:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762798.916714:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762798.916719:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1843 00000100:00080000:1.0:1787762798.916724:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762798.916729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762798.916733:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230656:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762798.916756:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762798.916772:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274230784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762798.916828:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230784:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787762799.492677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762799.940665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762800.516762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762800.964688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762801.540803:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762801.988607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787762801.988821:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787762801.988888:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787762801.988919:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787762801.988989:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762801.989122:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762798/real 1787762801] req@ffff9ac447840b40 x1874603274230784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762801.989158:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274230784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762801.989193:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762801.989203:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762801.989211:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274230784:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787762801.989258:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762798/real 1787762801] req@ffff9ac447842940 x1874603274230656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762801.989278:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274230656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762801.989288:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762801.989293:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762801.989298:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230656:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787762802.564647:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762803.012652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787762803.588667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762804.036701:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762804.036702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762804.036719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762804.036727:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762804.036741:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1848 00000100:00080000:0.0:1787762804.036756:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762804.036765:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762804.036814:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274230912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762804.036871:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762804.036878:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762804.036882:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762804.036891:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1848 00000100:00080000:0.0:1787762804.036900:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762804.036910:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762804.036944:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230912:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787762804.036969:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762804.036996:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274231040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762804.037089:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762804.037142:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762804/real 1787762804] req@ffff9ac451f870c0 x1874603274230912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762804.037167:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274230912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762804.037184:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762804.037191:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762804.037197:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274230912:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762804.037241:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231040:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762804.037273:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762804.037291:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762804/real 1787762804] req@ffff9ac451f84000 x1874603274231040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762804.037306:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274231040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762804.037316:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762804.037320:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762804.037324:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231040:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762804.612669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762805.060679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762805.636678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762806.084642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762806.660654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762807.108634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762807.684768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762808.132633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762808.708671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762809.156595:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762809.156602:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762809.156605:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762809.156614:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1853 00000100:00080000:0.0:1787762809.156634:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787762809.156636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762809.156641:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762809.156670:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274231168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762809.156690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762809.156693:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762809.156694:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762809.156697:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1853 00000100:00080000:0.0:1787762809.156699:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762809.156702:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762809.156710:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762809.156718:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274231296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762809.156788:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231168:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762809.156902:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762809.156983:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762809/real 1787762809] req@ffff9ac451f861c0 x1874603274231168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762809.157003:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274231168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762809.157013:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762809.157017:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762809.157021:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231168:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762809.157044:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231296:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762809.157080:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762809.157097:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762809/real 1787762809] req@ffff9ac451f84780 x1874603274231296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762809.157113:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274231296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762809.157124:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762809.157129:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762809.157133:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231296:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762809.732687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762810.180584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762810.756804:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762811.204592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762811.780643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762812.228585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762812.804718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762813.252631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762813.828663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762814.276624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762814.276632:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762814.276636:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762814.276645:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1859 00000100:00080000:0.0:1787762814.276654:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762814.276661:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762814.276668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787762814.276694:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274231424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762814.276716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762814.276718:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762814.276720:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762814.276723:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1859 00000100:00080000:0.0:1787762814.276726:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762814.276729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762814.276737:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762814.276747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274231552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762814.276755:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231424:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762814.276825:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762814.276858:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762814/real 1787762814] req@ffff9ac451f85680 x1874603274231424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762814.276871:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274231424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762814.276879:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762814.276882:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762814.276885:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762814.276905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231552:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762814.276919:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762814.276926:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762814/real 1787762814] req@ffff9ac451f843c0 x1874603274231552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762814.276934:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274231552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762814.276940:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762814.276942:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762814.276944:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231552:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762814.852655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762815.300617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762815.876646:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762816.324595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762816.900722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762817.348633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762817.924741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762818.372605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762818.948739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762819.396599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762819.396644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762819.396651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762819.396656:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762819.396683:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1864 00000100:00080000:1.0:1787762819.396697:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762819.396705:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762819.396747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274231680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762819.396775:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762819.396779:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762819.396780:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762819.396784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1864 00000100:00080000:1.0:1787762819.396787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762819.396790:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762819.396801:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762819.396811:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231680:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762819.396815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274231808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762819.396912:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762819.396954:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762819/real 1787762819] req@ffff9ac447843c00 x1874603274231680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762819.396976:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274231680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762819.396986:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762819.396990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762819.396993:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231680:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762819.397016:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231808:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762819.397033:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762819.397053:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762819/real 1787762819] req@ffff9ac447842940 x1874603274231808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762819.397062:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274231808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762819.397068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762819.397070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762819.397073:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274231808:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762819.972662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762820.420606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762820.996709:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762821.444594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762822.020662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762822.468596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762823.044697:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762823.492696:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762824.068719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762824.516685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762824.516698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762824.516704:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762824.516716:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1869 00000100:00080000:0.0:1787762824.516730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787762824.516757:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762824.516766:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762824.516815:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274231936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762824.516854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762824.516859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762824.516862:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762824.516868:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1869 00000100:00080000:0.0:1787762824.516875:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762824.516881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762824.516894:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762824.516914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231936:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762824.516915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274232064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762824.517035:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762824.517083:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762824/real 1787762824] req@ffff9ac451f843c0 x1874603274231936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762824.517102:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274231936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762824.517112:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762824.517130:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762824.517134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274231936:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762824.517167:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232064:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762824.517195:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762824.517211:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762824/real 1787762824] req@ffff9ac451f85680 x1874603274232064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762824.517222:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274232064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762824.517229:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762824.517232:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762824.517235:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232064:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762825.092752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762825.540675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762826.116712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762826.564601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762827.140638:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762827.588598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762828.164668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762828.612614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762829.188657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762829.636596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762829.636654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762829.636663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762829.636669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762829.636681:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1874 00000100:00080000:3.0:1787762829.636697:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762829.636706:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762829.636760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274232192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762829.636794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762829.636798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762829.636800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762829.636841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1874 00000100:00080000:3.0:1787762829.636847:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787762829.636853:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232192:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787762829.636857:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762829.636877:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762829.636895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274232320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762829.636970:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762829.637025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762829/real 1787762829] req@ffff9ac451c52d00 x1874603274232192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762829.637046:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274232192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762829.637071:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762829.637077:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762829.637081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232192:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762829.637131:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232320:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762829.637160:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762829.637174:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762829/real 1787762829] req@ffff9ac451c530c0 x1874603274232320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762829.637187:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274232320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762829.637196:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762829.637200:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762829.637204:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232320:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762830.212791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762830.660650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762831.236852:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762831.684682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762832.260743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762832.708641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762833.284759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762833.732703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762834.308775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762834.756614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762834.756626:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762834.756632:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762834.756637:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762834.756648:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1879 00000100:00080000:1.0:1787762834.756663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762834.756673:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762834.756728:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274232448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762834.756759:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762834.756763:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762834.756764:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787762834.756789:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232448:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787762834.756794:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1879 00000100:00080000:1.0:1787762834.756799:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762834.756802:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762834.756814:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762834.756827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274232576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762834.756873:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762834.756918:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762834/real 1787762834] req@ffff9ac447840b40 x1874603274232448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762834.756933:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274232448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762834.756954:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762834.756957:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762834.756961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232448:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762834.756985:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232576:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762834.757005:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762834.757014:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762834/real 1787762834] req@ffff9ac447842d00 x1874603274232576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762834.757023:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274232576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762834.757029:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762834.757031:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762834.757034:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232576:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762835.332744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762835.780600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762836.356678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762836.804640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762837.380723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762837.828703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762838.404773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762838.852592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762839.428792:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762839.876626:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762839.876630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762839.876640:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762839.876645:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762839.876656:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1884 00000100:00080000:1.0:1787762839.876670:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762839.876678:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762839.876727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274232704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762839.876754:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762839.876758:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762839.876759:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762839.876763:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1884 00000100:00080000:1.0:1787762839.876765:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762839.876768:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762839.876779:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762839.876788:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274232832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762839.876793:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232704:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762839.876867:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762839.876916:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762839/real 1787762839] req@ffff9ac4478430c0 x1874603274232704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762839.876938:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274232704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762839.876949:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762839.876954:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762839.876959:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232704:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762839.876990:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232832:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762839.877013:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762839.877024:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762839/real 1787762839] req@ffff9ac4478421c0 x1874603274232832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762839.877034:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274232832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762839.877040:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762839.877043:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762839.877045:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274232832:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762840.452742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762840.900615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762841.476697:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762841.924621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762842.500775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762842.948608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762843.524785:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762843.972730:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762844.548723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762844.996620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762844.996642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762844.996653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762844.996667:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762844.996677:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1889 00000100:00080000:1.0:1787762844.996689:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762844.996696:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762844.996734:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274232960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762844.996765:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762844.996769:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762844.996771:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762844.996775:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1889 00000100:00080000:1.0:1787762844.996778:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762844.996781:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762844.996791:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762844.996805:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274233088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762844.996841:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232960:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762844.996963:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762844.997044:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762844/real 1787762844] req@ffff9ac4478403c0 x1874603274232960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762844.997095:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274232960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762844.997111:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762844.997118:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762844.997124:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274232960:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762844.997134:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233088:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762844.997212:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762844.997226:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762844/real 1787762844] req@ffff9ac447841680 x1874603274233088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762844.997237:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274233088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762844.997245:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762844.997248:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762844.997251:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233088:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762845.572768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762846.020617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762846.596704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762847.044625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762847.620713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762848.068590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762848.644682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762849.092619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762849.668686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762850.116608:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787762850.116608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762850.116623:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762850.116657:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762850.116670:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1894 00000100:00080000:1.0:1787762850.116684:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762850.116695:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762850.116747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274233216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762850.116778:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762850.116781:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762850.116783:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762850.116787:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1894 00000100:00080000:1.0:1787762850.116790:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762850.116794:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762850.116805:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762850.116816:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274233344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762850.116857:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233216:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762850.116937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762850.116982:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762850/real 1787762850] req@ffff9ac447840780 x1874603274233216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762850.117028:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274233216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762850.117039:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762850.117043:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762850.117046:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233216:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762850.117052:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233344:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762850.117073:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762850.117083:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762850/real 1787762850] req@ffff9ac447840f00 x1874603274233344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762850.117093:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274233344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762850.117121:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762850.117125:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762850.117128:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233344:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762850.692640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762851.140591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762851.716639:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762852.164598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762852.740655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762853.188627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762853.764652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762854.212627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762854.788745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762855.236639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762855.236698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762855.236709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762855.236714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762855.236726:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1899 00000100:00080000:3.0:1787762855.236738:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762855.236748:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762855.236797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274233472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762855.236838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762855.236846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762855.236850:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762855.236859:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1899 00000100:00080000:3.0:1787762855.236868:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762855.236877:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762855.236896:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233472:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787762855.236909:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762855.236947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274233600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762855.236990:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762855.237046:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762855/real 1787762855] req@ffff9ac451c52940 x1874603274233472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762855.237123:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274233472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762855.237137:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762855.237143:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762855.237148:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233472:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762855.237182:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233600:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762855.237215:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762855.237231:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762855/real 1787762855] req@ffff9ac451c50b40 x1874603274233600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762855.237247:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274233600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762855.237257:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762855.237261:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762855.237266:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233600:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762855.812751:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762856.260631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762856.836707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762857.284633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762857.860672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762858.308600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762858.884677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762859.332610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762859.908659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762860.356613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762860.356613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762860.356643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762860.356648:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762860.356659:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1905 00000100:00080000:3.0:1787762860.356675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762860.356685:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762860.356736:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274233728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762860.356763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762860.356766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762860.356768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762860.356771:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1905 00000100:00080000:3.0:1787762860.356774:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762860.356779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762860.356791:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762860.356800:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233728:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762860.356804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274233856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762860.356889:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762860.356932:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762860/real 1787762860] req@ffff9ac451c50780 x1874603274233728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762860.356947:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274233728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762860.356957:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762860.356960:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762860.356963:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233728:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762860.356988:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233856:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762860.357004:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762860.357013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762860/real 1787762860] req@ffff9ac451c503c0 x1874603274233856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762860.357022:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274233856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762860.357028:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762860.357046:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762860.357048:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274233856:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762860.932686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762861.380672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762861.956678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762862.404600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762862.980770:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762863.428619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762864.004688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762864.452647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762865.028732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762865.476600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762865.476604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762865.476615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762865.476621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762865.476634:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1910 00000100:00080000:3.0:1787762865.476649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762865.476658:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762865.476705:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274233984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762865.476751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762865.476754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762865.476755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762865.476759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1910 00000100:00080000:3.0:1787762865.476761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762865.476765:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762865.476775:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787762865.476786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274234112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762865.476793:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233984:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762865.476926:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234112:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762866.052703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762866.500610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762867.076703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762867.524617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762868.100672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762868.548654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787762868.548914:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787762868.548968:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787762868.549035:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787762868.549085:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787762868.549153:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762865/real 1787762868] req@ffff9ac451c50f00 x1874603274234112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762868.549195:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274234112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762868.549211:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762868.549218:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762868.549223:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234112:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787762868.549234:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762865/real 1787762868] req@ffff9ac451c52580 x1874603274233984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762868.549250:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274233984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762868.549258:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762868.549263:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762868.549267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274233984:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787762869.124731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762869.572672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762870.148721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787762870.596673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762870.596675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787762870.596689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787762870.596696:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762870.596709:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1915 00000100:00080000:3.0:1787762870.596725:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762870.596736:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787762870.596783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274234240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787762870.596820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787762870.596825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787762870.596828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787762870.596834:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1915 00000100:00080000:3.0:1787762870.596838:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787762870.596844:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787762870.596873:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762870.596886:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274234240:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787762870.596897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274234368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762870.597023:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762870.597102:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762870/real 1787762870] req@ffff9ac451c52580 x1874603274234240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762870.597136:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274234240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762870.597155:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762870.597163:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762870.597172:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274234240:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762870.597214:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234368:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762870.597264:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762870.597293:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762870/real 1787762870] req@ffff9ac451c50f00 x1874603274234368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762870.597319:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274234368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762870.597336:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762870.597343:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762870.597350:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234368:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762871.172706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762871.620623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762872.196743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762872.644651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762873.220800:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762873.668653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762874.244799:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762874.692607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762875.268727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787762875.716670:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787762875.716683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762875.716683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762875.716690:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762875.716705:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1920 00000100:00080000:2.0:1787762875.716720:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762875.716731:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762875.716780:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274234496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762875.716808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762875.716812:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762875.716814:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762875.716817:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1920 00000100:00080000:2.0:1787762875.716820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762875.716823:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762875.716834:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762875.716846:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274234624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762875.716876:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274234496:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762875.716992:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762875.717057:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762875/real 1787762875] req@ffff9ac4523bcb40 x1874603274234496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762875.717095:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274234496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762875.717118:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762875.717126:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762875.717134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274234496:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762875.717169:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234624:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762875.717203:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762875.717220:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762875/real 1787762875] req@ffff9ac4523bcf00 x1874603274234624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762875.717235:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274234624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762875.717247:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762875.717251:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762875.717256:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234624:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762876.292713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762876.740623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762877.316702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762877.764615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762878.340726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762878.788626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762879.364740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762879.812693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762880.388734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762880.836691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762880.836732:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762880.836745:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762880.836752:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762880.836769:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1925 00000100:00080000:1.0:1787762880.836787:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762880.836797:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762880.836845:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274234752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762880.836884:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762880.836917:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762880.836921:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762880.836929:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1925 00000100:00080000:1.0:1787762880.836937:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762880.836943:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762880.836952:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274234752:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762880.836962:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762880.836988:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274234880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762880.837041:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762880.837101:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762880/real 1787762880] req@ffff9ac447840000 x1874603274234752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762880.837160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274234752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762880.837181:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762880.837187:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762880.837192:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274234752:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762880.837230:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234880:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762880.837264:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762880.837281:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762880/real 1787762880] req@ffff9ac447842580 x1874603274234880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762880.837297:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274234880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762880.837308:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762880.837312:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762880.837329:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274234880:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762881.412776:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762881.860638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762882.436711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762882.884727:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762883.460759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762883.908677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762884.484678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762884.932614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762885.508672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762885.956578:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762885.956587:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762885.956591:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762885.956600:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1930 00000100:00080000:1.0:1787762885.956610:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787762885.956611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762885.956618:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762885.956652:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274235008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762885.956679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762885.956682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762885.956684:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762885.956687:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1930 00000100:00080000:1.0:1787762885.956690:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762885.956693:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762885.956709:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762885.956711:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235008:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762885.956722:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274235136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762885.956769:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762885.956803:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762885/real 1787762885] req@ffff9ac447843840 x1874603274235008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762885.956828:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274235008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762885.956836:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762885.956839:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762885.956842:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235008:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762885.956874:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235136:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762885.956891:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762885.956900:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762885/real 1787762885] req@ffff9ac447841a40 x1874603274235136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762885.956908:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274235136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762885.956914:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762885.956928:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762885.956930:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235136:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762886.532752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762886.980658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762887.556636:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762888.004621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762888.580722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762889.028656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762889.604712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762890.052648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762890.628677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762891.076646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762891.076659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1787762891.076685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762891.076687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762891.076709:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1935 00000100:00080000:1.0:1787762891.076723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762891.076734:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762891.076784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274235264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762891.076845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762891.076851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762891.076854:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762891.076861:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1935 00000100:00080000:1.0:1787762891.076866:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762891.076880:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762891.076881:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235264:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787762891.076913:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762891.076937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274235392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762891.076986:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762891.077040:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762891/real 1787762891] req@ffff9ac447841e00 x1874603274235264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762891.077057:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274235264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762891.077067:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762891.077071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762891.077074:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235264:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762891.077100:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235392:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762891.077116:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762891.077125:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762891/real 1787762891] req@ffff9ac4478412c0 x1874603274235392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762891.077133:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274235392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762891.077154:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762891.077156:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762891.077159:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235392:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762891.652650:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762892.100625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762892.676719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762893.124592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762893.700849:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762894.148669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762894.724788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762895.172648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762895.748706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762896.196652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762896.196659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762896.196672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762896.196679:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762896.196693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1940 00000100:00080000:1.0:1787762896.196710:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762896.196722:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762896.196780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274235520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762896.196869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762896.196874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762896.196877:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762896.196883:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1940 00000100:00080000:1.0:1787762896.196889:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762896.196895:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762896.196910:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762896.196931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274235648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762896.196980:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235520:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762896.197079:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762896.197137:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762896/real 1787762896] req@ffff9ac447843480 x1874603274235520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762896.197161:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274235520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762896.197176:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762896.197182:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762896.197187:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235520:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762896.197196:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235648:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762896.197223:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762896.197237:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762896/real 1787762896] req@ffff9ac4478412c0 x1874603274235648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762896.197295:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274235648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762896.197313:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762896.197320:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762896.197327:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235648:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762896.772691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762897.220664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762897.796755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787762898.244657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762898.820761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787762899.268709:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762899.844740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787762900.292595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762900.868806:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762901.316691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762901.316709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787762901.316714:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762901.316716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762901.316733:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1946 00000100:00080000:1.0:1787762901.316777:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762901.316803:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762901.316871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274235776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762901.316911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762901.316916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762901.316919:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762901.316949:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1946 00000100:00080000:1.0:1787762901.316957:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762901.316964:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762901.316982:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762901.317005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274235904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762901.317014:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235776:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762901.317142:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762901.317207:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762901/real 1787762901] req@ffff9ac447841e00 x1874603274235776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762901.317232:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274235776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762901.317251:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762901.317258:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762901.317264:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274235776:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762901.317299:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235904:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762901.317329:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762901.317344:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762901/real 1787762901] req@ffff9ac447841a40 x1874603274235904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762901.317374:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274235904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762901.317385:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762901.317389:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762901.317393:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274235904:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762901.892793:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762902.340653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762902.916800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762903.364644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762903.940666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762904.388669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762904.964758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787762905.412674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762905.988779:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762906.436656:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762906.436670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762906.436675:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762906.436691:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1951 00000100:00080000:1.0:1787762906.436708:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762906.436742:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762906.436749:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787762906.436802:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274236032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762906.436838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762906.436843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762906.436846:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762906.436852:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1951 00000100:00080000:1.0:1787762906.436856:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762906.436864:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762906.436880:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762906.436902:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236032:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762906.436924:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274236160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762906.436974:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762906.437016:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762906/real 1787762906] req@ffff9ac447843840 x1874603274236032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762906.437030:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274236032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762906.437040:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762906.437043:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762906.437046:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236032:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762906.437066:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236160:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762906.437082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762906.437090:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762906/real 1787762906] req@ffff9ac447842580 x1874603274236160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762906.437098:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274236160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762906.437112:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762906.437115:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762906.437117:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236160:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762907.012811:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787762907.460728:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762908.036691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762908.484694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762909.060796:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762909.508671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762910.084767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762910.532678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762911.108851:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762911.556644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762911.556710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762911.556723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762911.556728:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762911.556741:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1956 00000100:00080000:1.0:1787762911.556755:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762911.556764:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762911.556813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274236288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762911.556851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762911.556856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762911.556859:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762911.556865:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1956 00000100:00080000:1.0:1787762911.556870:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762911.556875:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762911.556888:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762911.556904:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236288:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762911.556906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274236416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762911.557017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762911.557065:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762911/real 1787762911] req@ffff9ac447840000 x1874603274236288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762911.557081:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274236288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762911.557091:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762911.557095:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762911.557099:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236288:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762911.557124:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236416:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762911.557145:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762911.557159:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762911/real 1787762911] req@ffff9ac447840f00 x1874603274236416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762911.557173:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274236416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762911.557206:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762911.557209:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762911.557212:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236416:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762912.132750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762912.580661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762913.156772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762913.604625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762914.180705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762914.628618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762915.204744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762915.652609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762916.228767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762916.676680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762916.676707:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762916.676718:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762916.676723:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762916.676737:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1961 00000100:00080000:2.0:1787762916.676751:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762916.676762:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762916.676809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274236544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762916.676847:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762916.676852:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762916.676854:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762916.676860:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1961 00000100:00080000:2.0:1787762916.676864:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762916.676869:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762916.676883:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762916.676900:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274236672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762916.676924:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236544:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762916.677009:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762916.677079:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762916/real 1787762916] req@ffff9ac4523be580 x1874603274236544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762916.677103:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274236544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762916.677118:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762916.677123:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762916.677129:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236544:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762916.677160:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236672:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762916.677190:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762916.677205:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762916/real 1787762916] req@ffff9ac4523bc780 x1874603274236672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762916.677221:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274236672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762916.677232:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762916.677254:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762916.677258:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236672:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762917.252791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762917.700688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762918.276786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762918.724697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762919.300754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762919.748687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762920.324791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762920.772608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762921.348731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762921.796600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762921.796619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762921.796626:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762921.796631:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762921.796643:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1966 00000100:00080000:1.0:1787762921.796656:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762921.796664:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762921.796706:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274236800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762921.796731:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762921.796734:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762921.796736:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762921.796739:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1966 00000100:00080000:1.0:1787762921.796741:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762921.796744:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762921.796755:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762921.796764:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274236928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762921.796890:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236800:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762921.796988:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762921.797047:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762921/real 1787762921] req@ffff9ac447840780 x1874603274236800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762921.797072:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274236800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762921.797088:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762921.797093:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762921.797098:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274236800:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762921.797107:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236928:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762921.797151:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762921.797167:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762921/real 1787762921] req@ffff9ac447841680 x1874603274236928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762921.797184:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274236928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762921.797195:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762921.797206:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762921.797211:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274236928:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762922.372672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762922.820671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762923.396690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762923.844622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762924.420662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762924.868623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762925.444709:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762925.892644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762926.468687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762926.916628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762926.916630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762926.916639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762926.916644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762926.916655:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1971 00000100:00080000:0.0:1787762926.916667:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762926.916674:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762926.916718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274237056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762926.916743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762926.916746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762926.916747:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762926.916751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1971 00000100:00080000:0.0:1787762926.916753:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762926.916756:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762926.916767:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762926.916779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274237184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762926.916779:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237056:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787762926.916868:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762926.916912:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762926/real 1787762926] req@ffff9ac451f84780 x1874603274237056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762926.916949:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274237056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762926.916958:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762926.916963:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762926.916968:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762926.916999:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237184:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762926.917025:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762926.917036:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762926/real 1787762926] req@ffff9ac451f861c0 x1874603274237184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762926.917045:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274237184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762926.917051:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762926.917054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762926.917056:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237184:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762927.492666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762927.940676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762928.516748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762928.964664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762929.540780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762929.988617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762930.564726:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762931.012630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762931.588677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762932.036614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762932.036679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762932.036692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762932.036730:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762932.036747:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1976 00000100:00080000:0.0:1787762932.036764:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762932.036776:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762932.036830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274237312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762932.036877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762932.036886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762932.036890:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762932.036899:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1976 00000100:00080000:0.0:1787762932.036907:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762932.036915:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762932.036931:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762932.036948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237312:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762932.036955:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274237440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787762932.037102:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237440:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762932.612735:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762933.060672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762933.636662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762934.084598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762934.660675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762935.108711:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787762935.108981:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787762935.109031:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787762935.109053:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787762935.109095:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787762935.109185:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762932/real 1787762935] req@ffff9ac451f870c0 x1874603274237440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762935.109239:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274237440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762935.109254:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762935.109261:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762935.109267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237440:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787762935.109276:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762932/real 1787762935] req@ffff9ac451f84000 x1874603274237312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762935.109293:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274237312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762935.109303:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762935.109307:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762935.109311:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237312:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787762935.684772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762936.132708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762936.708657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762937.156633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762937.156757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762937.156770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762937.156776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762937.156789:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1981 00000100:00080000:1.0:1787762937.156803:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762937.156813:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762937.156860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274237568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762937.156912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762937.156942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762937.156947:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762937.156958:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1981 00000100:00080000:1.0:1787762937.156969:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787762937.156970:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237568:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787762937.156988:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762937.157008:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762937.157044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274237696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787762937.157080:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762937.157152:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762937/real 1787762937] req@ffff9ac4478403c0 x1874603274237568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762937.157186:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274237568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762937.157207:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762937.157232:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762937.157241:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237568:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762937.157283:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237696:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762937.157335:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762937.157362:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762937/real 1787762937] req@ffff9ac4478421c0 x1874603274237696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762937.157386:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274237696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762937.157405:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762937.157412:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762937.157419:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237696:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762937.732725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762938.180605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762938.756754:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762939.204648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762939.780878:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762940.228611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762940.804995:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762941.252643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762941.828712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762942.276637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787762942.276663:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762942.276673:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787762942.276704:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762942.276718:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1987 00000100:00080000:2.0:1787762942.276733:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762942.276743:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787762942.276794:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274237824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787762942.276834:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787762942.276839:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787762942.276842:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787762942.276847:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1987 00000100:00080000:2.0:1787762942.276851:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787762942.276855:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787762942.276868:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762942.276884:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274237952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762942.276954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237824:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762942.277075:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762942.277141:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762942/real 1787762942] req@ffff9ac4523bd680 x1874603274237824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762942.277166:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274237824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762942.277180:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762942.277186:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762942.277191:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274237824:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762942.277212:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237952:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762942.277246:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762942.277261:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762942/real 1787762942] req@ffff9ac4523bc000 x1874603274237952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762942.277276:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274237952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762942.277285:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762942.277290:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762942.277294:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274237952:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762942.852791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762943.300678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762943.877722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762944.324642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762944.900815:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762945.348641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762945.924701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762946.372615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762946.948658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762947.396621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762947.396633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762947.396640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762947.396645:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762947.396656:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1992 00000100:00080000:1.0:1787762947.396670:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762947.396678:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762947.396727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274238080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762947.396754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762947.396757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762947.396758:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762947.396762:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1992 00000100:00080000:1.0:1787762947.396784:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762947.396788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762947.396799:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762947.396810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274238208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762947.396835:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238080:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762947.396915:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762947.396957:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762947/real 1787762947] req@ffff9ac4478430c0 x1874603274238080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762947.396971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274238080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762947.396980:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762947.396983:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762947.396986:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238080:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762947.396992:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238208:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762947.397019:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762947.397027:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762947/real 1787762947] req@ffff9ac447842d00 x1874603274238208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762947.397036:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274238208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762947.397041:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762947.397044:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762947.397046:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238208:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762947.972670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762948.420621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762948.996656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762949.444678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762950.020767:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762950.468695:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762951.044645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762951.492649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762952.068663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762952.516666:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787762952.516672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762952.516682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762952.516688:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762952.516717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 1997 00000100:00080000:1.0:1787762952.516729:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762952.516738:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762952.516805:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274238336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762952.516835:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762952.516839:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762952.516841:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762952.516845:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 1997 00000100:00080000:1.0:1787762952.516848:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762952.516852:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762952.516864:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762952.516876:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274238464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762952.516909:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238336:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762952.517028:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762952.517084:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762952/real 1787762952] req@ffff9ac447840b40 x1874603274238336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762952.517107:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274238336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762952.517120:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762952.517126:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762952.517131:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762952.517184:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238464:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762952.517207:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762952.517228:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762952/real 1787762952] req@ffff9ac447842940 x1874603274238464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762952.517256:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274238464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762952.517262:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762952.517264:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762952.517267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238464:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762953.092680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762953.540664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762954.116692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762954.564662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762955.140754:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762955.588591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762956.164712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762956.612654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762957.188706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762957.636626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762957.636665:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762957.636677:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762957.636683:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762957.636695:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2002 00000100:00080000:1.0:1787762957.636709:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762957.636718:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762957.636764:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274238592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762957.636802:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762957.636807:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762957.636810:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762957.636815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2002 00000100:00080000:1.0:1787762957.636820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762957.636824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762957.636837:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762957.636853:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274238720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762957.636903:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238592:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762957.636999:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762957.637047:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762957/real 1787762957] req@ffff9ac447843c00 x1874603274238592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762957.637063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274238592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762957.637074:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762957.637077:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762957.637081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238592:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762957.637087:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238720:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762957.637104:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762957.637112:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762957/real 1787762957] req@ffff9ac447842940 x1874603274238720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762957.637153:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274238720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762957.637159:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762957.637161:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762957.637164:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238720:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762958.212679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762958.660622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762959.236675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762959.684596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762960.260778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762960.708633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762961.284663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762961.732596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762962.308685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762962.756634:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787762962.756645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762962.756648:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762962.756686:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762962.756700:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2007 00000100:00080000:1.0:1787762962.756717:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762962.756727:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762962.756778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274238848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762962.756817:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762962.756821:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762962.756823:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762962.756828:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2007 00000100:00080000:1.0:1787762962.756832:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762962.756836:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762962.756849:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762962.756863:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274238976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787762962.756883:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238848:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787762962.756980:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762962.757034:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762962/real 1787762962] req@ffff9ac447840b40 x1874603274238848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762962.757055:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274238848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762962.757068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762962.757073:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762962.757077:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274238848:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762962.757106:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238976:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787762962.757130:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787762962.757142:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762962/real 1787762962] req@ffff9ac447842d00 x1874603274238976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787762962.757174:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274238976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787762962.757183:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787762962.757187:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787762962.757190:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274238976:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762963.332648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762963.780657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762964.356671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762964.804602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762965.380717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762965.828718:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762966.404715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762966.852661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762967.428717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762967.876660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762967.876699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762967.876712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762967.876718:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762967.876742:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2012 00000100:00080000:0.0:1787762967.876757:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762967.876766:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762967.876813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274239104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762967.876852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762967.876857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762967.876860:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762967.876865:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2012 00000100:00080000:0.0:1787762967.876869:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762967.876875:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762967.876888:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762967.876913:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787762967.876937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274239232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787762967.877005:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762967.877056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762967/real 1787762967] req@ffff9ac451f87c00 x1874603274239104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762967.877079:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274239104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762967.877118:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762967.877126:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762967.877135:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762967.877175:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274239232:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787762967.877224:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787762967.877247:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762967/real 1787762967] req@ffff9ac451f84b40 x1874603274239232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787762967.877263:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274239232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787762967.877275:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787762967.877279:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787762967.877283:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274239232:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762968.452782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762968.900650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762969.476766:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762969.924642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762970.500774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762970.948703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762971.524774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762971.972666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762972.548699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762972.996672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787762972.996678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762972.996688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762972.996693:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762972.996708:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2017 00000100:00080000:0.0:1787762972.996723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762972.996737:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762972.996795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274239360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762972.996838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762972.996842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762972.996843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762972.996848:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2017 00000100:00080000:0.0:1787762972.996852:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762972.996857:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762972.996868:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762972.996882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274239488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787762972.996934:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239360:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787762972.997022:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762972.997075:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762972/real 1787762972] req@ffff9ac451f84b40 x1874603274239360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762972.997097:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274239360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762972.997111:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762972.997116:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762972.997143:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239360:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762972.997152:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274239488:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787762972.997181:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762972.997194:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762972/real 1787762972] req@ffff9ac451f87c00 x1874603274239488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762972.997203:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274239488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762972.997210:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762972.997212:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762972.997214:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274239488:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762973.572693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762974.020637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762974.596794:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762975.044641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762975.620774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762976.068623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762976.644748:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762977.092673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762977.668674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762978.116610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762978.116619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762978.116625:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762978.116629:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762978.116637:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2022 00000100:00080000:1.0:1787762978.116646:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762978.116652:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762978.116695:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274239616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787762978.116720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762978.116723:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762978.116726:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762978.116730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2022 00000100:00080000:1.0:1787762978.116734:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762978.116739:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762978.116749:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762978.116763:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274239744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787762978.116794:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239616:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787762978.116860:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762978.116894:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762978/real 1787762978] req@ffff9ac4478430c0 x1874603274239616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762978.116907:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274239616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762978.116914:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762978.116917:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762978.116920:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239616:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762978.116925:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274239744:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787762978.116940:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762978.116956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762978/real 1787762978] req@ffff9ac4478421c0 x1874603274239744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762978.116965:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274239744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762978.116970:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762978.116973:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762978.116975:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274239744:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762978.692720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762979.140598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762979.716647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762980.164600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762980.740681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762981.188614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762981.764668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762982.212601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762982.788657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762983.236589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762983.236598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762983.236602:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762983.236614:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2027 00000100:00080000:1.0:1787762983.236625:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762983.236633:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762983.236670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274239872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787762983.236682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762983.236697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762983.236701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762983.236703:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762983.236724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2027 00000100:00080000:1.0:1787762983.236732:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762983.236736:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762983.236745:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787762983.236758:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239872:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787762983.236764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274240000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787762983.236827:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762983.236861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762983/real 1787762983] req@ffff9ac4478403c0 x1874603274239872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762983.236876:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274239872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762983.236884:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762983.236887:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762983.236890:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274239872:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762983.236908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240000:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787762983.236923:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762983.236930:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762983/real 1787762983] req@ffff9ac447841680 x1874603274240000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762983.236939:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274240000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762983.236957:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762983.236960:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762983.236963:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240000:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762983.812695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762984.260672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762984.836751:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762985.284634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762985.860786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762986.308606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762986.884774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762987.332641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762987.908690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787762988.356603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762988.356612:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787762988.356616:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762988.356625:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2033 00000100:00080000:1.0:1787762988.356635:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762988.356643:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787762988.356680:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274240128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787762988.356685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787762988.356713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787762988.356718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787762988.356720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787762988.356725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2033 00000100:00080000:1.0:1787762988.356728:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787762988.356735:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787762988.356746:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787762988.356789:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274240256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787762988.356805:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240128:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787762988.356881:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762988.356933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762988/real 1787762988] req@ffff9ac447840780 x1874603274240128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762988.356980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274240128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762988.356997:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762988.357003:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762988.357008:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240128:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762988.357043:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240256:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787762988.357079:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762988.357096:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762988/real 1787762988] req@ffff9ac447840f00 x1874603274240256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762988.357111:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274240256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762988.357122:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762988.357127:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762988.357131:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240256:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762988.932814:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762989.380666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762989.956724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762990.404662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762990.980795:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762991.428676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762992.004787:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762992.452618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762993.028817:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762993.476631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762993.476684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762993.476697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762993.476703:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762993.476716:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2038 00000100:00080000:0.0:1787762993.476730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762993.476739:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762993.476789:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274240384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762993.476831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762993.476837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762993.476840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762993.476845:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2038 00000100:00080000:0.0:1787762993.476849:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762993.476855:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762993.476868:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762993.476885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274240512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787762993.476909:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240384:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787762993.476999:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762993.477052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762993/real 1787762993] req@ffff9ac451f84000 x1874603274240384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762993.477072:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274240384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762993.477084:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762993.477089:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762993.477092:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240384:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762993.477134:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240512:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787762993.477158:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787762993.477172:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762993/real 1787762993] req@ffff9ac451f870c0 x1874603274240512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787762993.477186:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274240512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787762993.477196:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787762993.477200:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787762993.477205:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240512:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787762994.052690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762994.500612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762995.076818:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762995.524622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762996.100642:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762996.548610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787762997.124712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762997.572638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787762998.148680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787762998.596603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787762998.596605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787762998.596615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787762998.596620:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762998.596632:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2043 00000100:00080000:0.0:1787762998.596647:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762998.596657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787762998.596714:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274240640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787762998.596746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787762998.596750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787762998.596751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787762998.596755:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2043 00000100:00080000:0.0:1787762998.596759:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787762998.596763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787762998.596774:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787762998.596787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274240768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787762998.596810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240640:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787762998.596936:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240768:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787762999.172765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787762999.620707:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763000.196667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763000.644666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763001.220693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763001.668813:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787763001.668935:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787763001.668994:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787763001.669017:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787763001.669077:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787763001.669113:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762998/real 1787763001] req@ffff9ac451f84780 x1874603274240768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763001.669145:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274240768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763001.669160:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763001.669167:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763001.669173:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274240768:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787763001.669182:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787762998/real 1787763001] req@ffff9ac451f861c0 x1874603274240640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763001.669196:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274240640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763001.669206:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763001.669209:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763001.669213:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240640:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787763002.244683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763002.692665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763003.268650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763003.716615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763003.716626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763003.716632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763003.716644:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2048 00000100:00080000:0.0:1787763003.716657:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763003.716683:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763003.716696:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787763003.716726:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274240896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763003.716756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763003.716760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763003.716761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763003.716765:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2048 00000100:00080000:0.0:1787763003.716768:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763003.716771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763003.716783:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763003.716798:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274241024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787763003.716803:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240896:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787763003.716901:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763003.716958:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763003/real 1787763003] req@ffff9ac451f85680 x1874603274240896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763003.716980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274240896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763003.716994:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763003.716999:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763003.717004:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274240896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763003.717036:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241024:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763003.717077:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763003.717093:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763003/real 1787763003] req@ffff9ac451f843c0 x1874603274241024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763003.717107:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274241024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763003.717116:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763003.717120:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763003.717123:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241024:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763004.292653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763004.740731:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763005.316660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763005.764598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763006.340688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763006.788615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763007.364683:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763007.812706:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763008.388698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763008.836642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763008.836662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763008.836669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763008.836689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763008.836700:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2053 00000100:00080000:1.0:1787763008.836721:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763008.836730:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763008.836772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274241152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763008.836799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763008.836802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763008.836804:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763008.836808:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2053 00000100:00080000:1.0:1787763008.836811:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763008.836814:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763008.836826:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763008.836831:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241152:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763008.836844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274241280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763008.836988:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763008.837206:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763008/real 1787763008] req@ffff9ac447840000 x1874603274241152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763008.837228:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274241152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763008.837239:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763008.837244:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763008.837248:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241152:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763008.837271:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241280:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763008.837292:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763008.837303:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763008/real 1787763008] req@ffff9ac447842580 x1874603274241280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763008.837312:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274241280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763008.837318:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763008.837321:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763008.837323:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241280:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763009.412795:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763009.860637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763010.436717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763010.884642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763011.460727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763011.908615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763012.484714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763012.932635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763013.508652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763013.956616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763013.956623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763013.956628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763013.956634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763013.956645:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2058 00000100:00080000:0.0:1787763013.956677:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763013.956684:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763013.956730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274241408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763013.956755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763013.956758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763013.956760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763013.956764:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2058 00000100:00080000:0.0:1787763013.956766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763013.956769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763013.956780:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763013.956790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274241536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763013.956797:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241408:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763013.956884:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763013.956932:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763013/real 1787763013] req@ffff9ac451f843c0 x1874603274241408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763013.956949:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274241408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763013.956958:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763013.956962:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763013.956965:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241408:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763013.957003:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241536:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763013.957028:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763013.957043:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763013/real 1787763013] req@ffff9ac451f85680 x1874603274241536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763013.957058:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274241536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763013.957066:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763013.957069:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763013.957071:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241536:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763014.532696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763014.980623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763015.556686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763016.004589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763016.580829:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763017.028632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763017.604718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763018.052656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763018.628647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763019.076611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763019.076629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763019.076637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763019.076643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763019.076656:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2063 00000100:00080000:1.0:1787763019.076672:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763019.076683:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763019.076755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274241664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763019.076796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763019.076800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763019.076803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763019.076808:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2063 00000100:00080000:1.0:1787763019.076814:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763019.076820:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763019.076833:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763019.076835:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763019.076852:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274241792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763019.076924:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763019.076986:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763019/real 1787763019] req@ffff9ac447843840 x1874603274241664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763019.077016:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274241664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763019.077026:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763019.077030:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763019.077033:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763019.077057:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241792:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763019.077076:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763019.077085:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763019/real 1787763019] req@ffff9ac447841a40 x1874603274241792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763019.077094:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274241792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763019.077100:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763019.077103:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763019.077105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274241792:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763019.652648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763020.100663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763020.676745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763021.124645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763021.700717:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763022.148640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763022.724763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763023.172704:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763023.748721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763024.196691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763024.196705:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763024.196712:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763024.196728:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2068 00000100:00100000:0.0:1787763024.196747:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763024.196747:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763024.196765:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763024.196824:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274241920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763024.196862:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763024.196867:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763024.196870:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763024.196875:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2068 00000100:00080000:2.0:1787763024.196933:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787763024.196944:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241920:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787763024.196954:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763024.196977:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763024.197001:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274242048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763024.197034:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763024.197088:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763024/real 1787763024] req@ffff9ac4523be940 x1874603274241920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763024.197104:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274241920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763024.197139:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763024.197143:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763024.197146:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274241920:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763024.197168:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242048:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763024.197187:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763024.197196:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763024/real 1787763024] req@ffff9ac4523bc3c0 x1874603274242048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763024.197204:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274242048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763024.197209:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763024.197212:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763024.197214:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242048:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763024.772691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763025.220601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763025.796684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763026.244652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763026.820693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763027.268590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763027.844727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763028.292686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763028.868762:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763029.316628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763029.316681:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763029.316694:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763029.316699:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763029.316712:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2074 00000100:00080000:1.0:1787763029.316726:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763029.316735:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763029.316791:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274242176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763029.316830:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763029.316855:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763029.316859:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763029.316889:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2074 00000100:00100000:3.0:1787763029.316899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242176:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763029.316902:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763029.316919:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763029.316935:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763029.316960:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274242304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763029.317002:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763029.317056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763029/real 1787763029] req@ffff9ac447841e00 x1874603274242176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763029.317075:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274242176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763029.317086:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763029.317091:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763029.317109:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242176:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763029.317136:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242304:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763029.317165:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763029.317176:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763029/real 1787763029] req@ffff9ac4478412c0 x1874603274242304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763029.317185:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274242304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763029.317191:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763029.317194:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763029.317196:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242304:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763029.892666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763030.340623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763030.916750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763031.364616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763031.940764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763032.388620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763032.964689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763033.412624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763033.988676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763034.436635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763034.436635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763034.436648:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763034.436655:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763034.436666:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2079 00000100:00080000:1.0:1787763034.436690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763034.436700:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763034.436753:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274242432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763034.436784:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763034.436789:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763034.436791:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763034.436795:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2079 00000100:00080000:1.0:1787763034.436798:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763034.436801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763034.436813:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763034.436828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274242560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763034.436861:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242432:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763034.436963:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763034.437026:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763034/real 1787763034] req@ffff9ac447843480 x1874603274242432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763034.437047:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274242432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763034.437059:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763034.437063:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763034.437066:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242432:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763034.437095:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242560:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763034.437128:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763034.437140:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763034/real 1787763034] req@ffff9ac4478412c0 x1874603274242560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763034.437152:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274242560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763034.437160:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763034.437163:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763034.437166:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242560:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763035.012642:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763035.460594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763036.036706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763036.484610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763037.060688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763037.508660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763038.084784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763038.532668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763039.108738:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763039.556669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763039.556676:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763039.556689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763039.556694:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763039.556705:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2084 00000100:00080000:1.0:1787763039.556716:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763039.556724:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763039.556760:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274242688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763039.556783:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763039.556787:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763039.556788:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763039.556792:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2084 00000100:00080000:1.0:1787763039.556794:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763039.556808:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763039.556818:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763039.556831:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274242816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763039.556842:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242688:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763039.556901:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763039.556947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763039/real 1787763039] req@ffff9ac447841e00 x1874603274242688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763039.556966:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274242688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763039.556977:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763039.556981:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763039.556984:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763039.557003:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242816:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763039.557018:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763039.557041:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763039/real 1787763039] req@ffff9ac447841a40 x1874603274242816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763039.557050:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274242816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763039.557057:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763039.557059:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763039.557062:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274242816:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763040.132698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763040.580685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763041.156660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763041.604610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763042.180649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763042.628607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763043.204825:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763043.652598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763044.228658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763044.676639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763044.676703:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763044.676716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763044.676721:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763044.676735:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2089 00000100:00080000:1.0:1787763044.676750:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763044.676781:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763044.676851:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274242944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763044.676892:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763044.676897:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763044.676900:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763044.676912:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2089 00000100:00080000:1.0:1787763044.676918:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763044.676924:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763044.676937:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763044.676963:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274243072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763044.676996:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242944:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763044.677083:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763044.677137:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763044/real 1787763044] req@ffff9ac447843840 x1874603274242944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763044.677163:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274242944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763044.677178:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763044.677184:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763044.677190:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274242944:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763044.677226:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243072:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763044.677258:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763044.677273:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763044/real 1787763044] req@ffff9ac447842580 x1874603274243072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763044.677704:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274243072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763044.677719:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763044.677726:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763044.677732:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243072:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763045.252702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763045.700608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763046.276731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763046.724634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763047.300733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763047.748631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763048.324732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763048.772646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763049.348727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763049.796598:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763049.796607:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763049.796611:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763049.796620:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2094 00000100:00080000:1.0:1787763049.796629:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763049.796636:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763049.796669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274243200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763049.796693:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763049.796697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763049.796698:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763049.796701:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763049.796709:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2094 00000100:00080000:1.0:1787763049.796718:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763049.796748:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763049.796750:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243200:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763049.796760:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763049.796778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274243328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763049.796814:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763049.796857:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763049/real 1787763049] req@ffff9ac447840000 x1874603274243200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763049.796872:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274243200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763049.796882:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763049.796887:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763049.796891:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243200:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763049.796915:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243328:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763049.796939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763049.796950:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763049/real 1787763049] req@ffff9ac447840f00 x1874603274243328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763049.796973:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274243328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763049.796980:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763049.796982:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763049.796985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243328:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763050.372788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763050.820606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763051.396809:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763051.844672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763052.420756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763052.868588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763053.444657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763053.892635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763054.468655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763054.916601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763054.916624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763054.916630:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763054.916635:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763054.916645:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2099 00000100:00080000:3.0:1787763054.916657:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763054.916664:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763054.916718:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274243456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763054.916744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763054.916746:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763054.916748:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763054.916751:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2099 00000100:00080000:3.0:1787763054.916754:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763054.916757:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763054.916768:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763054.916781:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763054.916798:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274243584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763054.916854:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763054.916893:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763054/real 1787763054] req@ffff9ac451c503c0 x1874603274243456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763054.916907:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274243456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763054.916916:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763054.916919:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763054.916922:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763054.916944:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243584:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763054.916959:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763054.916968:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763054/real 1787763054] req@ffff9ac451c50780 x1874603274243584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763054.916976:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274243584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763054.916981:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763054.916984:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763054.916986:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243584:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763055.492665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763055.940668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763056.516668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763056.964617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763057.540746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763057.988739:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763058.564659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763059.012609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763059.588687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763060.036619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763060.036641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763060.036653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763060.036668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763060.036680:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2104 00000100:00080000:0.0:1787763060.036694:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763060.036713:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763060.036760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274243712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763060.036785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763060.036788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763060.036790:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763060.036793:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2104 00000100:00080000:0.0:1787763060.036795:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763060.036798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763060.036810:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763060.036824:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243712:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763060.036848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274243840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763060.036912:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763060.036955:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763060/real 1787763060] req@ffff9ac451f861c0 x1874603274243712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763060.036971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274243712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763060.036980:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763060.036983:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763060.036987:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243712:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763060.037010:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243840:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763060.037027:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763060.037036:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763060/real 1787763060] req@ffff9ac451f84780 x1874603274243840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763060.037044:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274243840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763060.037051:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763060.037053:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763060.037055:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274243840:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763060.612778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763061.060627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763061.636719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763062.084695:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763062.660761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763063.108688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763063.684669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763064.132625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763064.708669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763065.156612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763065.156660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763065.156669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763065.156672:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763065.156693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2109 00000100:00080000:1.0:1787763065.156702:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763065.156708:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763065.156739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274243968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763065.156763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763065.156766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763065.156768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763065.156771:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2109 00000100:00080000:1.0:1787763065.156773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763065.156775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763065.156783:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763065.156792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274244096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763065.156816:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763065.156901:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244096:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763065.732704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763066.180728:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763066.756676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763067.204733:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763067.780709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763068.228901:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763068.229071:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763068.229168:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763068.229192:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763068.229287:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:3.0:1787763068.229414:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763065/real 1787763068] req@ffff9ac447840780 x1874603274243968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763068.229451:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274243968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763068.229474:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763068.229482:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763068.229491:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274243968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763068.229504:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763065/real 1787763068] req@ffff9ac447841680 x1874603274244096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763068.229523:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274244096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763068.229700:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763068.229743:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763068.229753:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244096:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763068.804699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763069.252757:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763069.828684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763070.276637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763070.276650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763070.276656:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763070.276669:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2115 00000100:00080000:0.0:1787763070.276685:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763070.276691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763070.276697:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763070.276742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274244224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763070.276768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763070.276771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763070.276773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763070.276776:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2115 00000100:00080000:0.0:1787763070.276779:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763070.276783:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763070.276793:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763070.276803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274244352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763070.276809:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244224:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763070.276899:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763070.276949:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763070/real 1787763070] req@ffff9ac451f870c0 x1874603274244224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763070.276980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274244224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763070.276992:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763070.276996:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763070.277001:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244224:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763070.277030:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244352:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763070.277058:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763070.277072:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763070/real 1787763070] req@ffff9ac451f84000 x1874603274244352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763070.277086:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274244352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763070.277096:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763070.277100:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763070.277104:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244352:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763070.852736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763071.300743:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763071.876645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763072.324611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763072.900695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763073.348650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763073.924649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763074.372598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763074.948679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763075.396651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763075.396673:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763075.396679:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763075.396683:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763075.396691:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2120 00000100:00080000:3.0:1787763075.396700:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763075.396706:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763075.396740:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274244480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763075.396771:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763075.396776:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763075.396779:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763075.396784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2120 00000100:00080000:3.0:1787763075.396788:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763075.396793:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763075.396802:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763075.396810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244480:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763075.396815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274244608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763075.396868:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763075.396908:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763075/real 1787763075] req@ffff9ac451c50b40 x1874603274244480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763075.396922:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274244480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763075.396941:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763075.396944:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763075.396947:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244480:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763075.396965:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244608:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763075.396982:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763075.396991:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763075/real 1787763075] req@ffff9ac451c52940 x1874603274244608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763075.396999:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274244608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763075.397005:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763075.397007:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763075.397009:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244608:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763075.972659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763076.420615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763076.996647:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763077.444637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763078.020632:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763078.468616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763079.044693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763079.492644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763080.068642:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763080.516596:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763080.516605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763080.516607:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763080.516613:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763080.516624:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2125 00000100:00080000:3.0:1787763080.516646:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763080.516654:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763080.516686:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274244736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763080.516718:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763080.516722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763080.516725:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763080.516731:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2125 00000100:00080000:3.0:1787763080.516736:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763080.516741:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763080.516746:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244736:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787763080.516752:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763080.516769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274244864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763080.516803:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763080.516842:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763080/real 1787763080] req@ffff9ac451c530c0 x1874603274244736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763080.516878:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274244736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763080.516887:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763080.516890:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763080.516894:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244736:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763080.516916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244864:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763080.516936:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763080.516946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763080/real 1787763080] req@ffff9ac451c52d00 x1874603274244864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763080.516954:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274244864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763080.516961:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763080.516963:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763080.516966:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274244864:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763081.092699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763081.540604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763082.116649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763082.564597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763083.140657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763083.588618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763084.164669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763084.612579:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763085.188659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763085.636586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763085.636586:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763085.636595:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763085.636599:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763085.636608:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2130 00000100:00080000:3.0:1787763085.636618:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763085.636626:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763085.636661:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274244992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763085.636683:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763085.636699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763085.636702:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763085.636707:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2130 00000100:00080000:3.0:1787763085.636711:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763085.636713:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244992:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787763085.636715:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763085.636725:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763085.636738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274245120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763085.636776:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763085.636817:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763085/real 1787763085] req@ffff9ac451c52d00 x1874603274244992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763085.636838:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274244992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763085.636846:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763085.636849:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763085.636852:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274244992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763085.636873:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245120:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763085.636891:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763085.636900:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763085/real 1787763085] req@ffff9ac451c530c0 x1874603274245120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763085.636916:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274245120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763085.636923:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763085.636926:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763085.636928:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245120:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763086.212645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763086.660599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763087.236660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763087.684610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763088.260723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763088.708679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763089.284657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763089.732610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763090.308655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763090.756603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763090.756620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763090.756627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763090.756632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763090.756643:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2135 00000100:00080000:1.0:1787763090.756657:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763090.756665:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763090.756712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274245248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763090.756739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763090.756763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763090.756765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763090.756769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274245248:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763090.756773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2135 00000100:00080000:1.0:1787763090.756779:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763090.756782:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763090.756793:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763090.756805:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274245376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763090.756852:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763090.756904:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763090/real 1787763090] req@ffff9ac4478403c0 x1874603274245248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763090.756941:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274245248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763090.756951:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763090.756955:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763090.756958:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274245248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763090.756983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245376:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763090.757002:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763090.757012:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763090/real 1787763090] req@ffff9ac4478421c0 x1874603274245376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763090.757021:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274245376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763090.757027:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763090.757029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763090.757032:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245376:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763091.332657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763091.780601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763092.356728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763092.804624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763093.380745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763093.828624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763094.404731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763094.852625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763095.428739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763095.876606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763095.876624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763095.876631:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763095.876636:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763095.876649:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2140 00000100:00080000:0.0:1787763095.876666:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763095.876676:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763095.876728:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274245504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763095.876755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763095.876758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763095.876760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763095.876763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2140 00000100:00080000:0.0:1787763095.876767:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763095.876771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763095.876782:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763095.876797:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274245504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763095.876818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274245632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763095.876898:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763095.876960:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763095/real 1787763095] req@ffff9ac451f87c00 x1874603274245504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763095.876991:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274245504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763095.877013:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763095.877018:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763095.877024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274245504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763095.877056:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245632:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763095.877082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763095.877097:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763095/real 1787763095] req@ffff9ac451f84b40 x1874603274245632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763095.877111:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274245632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763095.877121:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763095.877125:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763095.877129:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245632:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763096.452654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763096.900624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763097.476676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763097.924642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763098.500728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763098.948656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763099.524658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763099.972607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763100.548701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763100.996627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763100.996638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763100.996645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763100.996651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763100.996662:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2145 00000100:00080000:1.0:1787763100.996675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763100.996683:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763100.996730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274245760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763100.996756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763100.996759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763100.996761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763100.996764:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2145 00000100:00080000:1.0:1787763100.996766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763100.996769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763100.996779:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763100.996788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274245888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763100.996798:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274245760:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763100.996886:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763100.996949:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763100/real 1787763100] req@ffff9ac4478430c0 x1874603274245760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763100.996980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274245760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763100.996994:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763100.996999:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763100.997004:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274245760:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763100.997031:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245888:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763100.997057:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763100.997072:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763100/real 1787763100] req@ffff9ac447842d00 x1874603274245888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763100.997087:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274245888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763100.997097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763100.997101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763100.997105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274245888:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763101.572660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763102.020609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763102.596678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763103.044627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763103.620661:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763104.068595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763104.644680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763105.092611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763105.668677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763106.116602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763106.116613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763106.116621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763106.116627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763106.116642:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2150 00000100:00080000:1.0:1787763106.116658:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763106.116668:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763106.116723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274246016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763106.116750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763106.116753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763106.116755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763106.116758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2150 00000100:00080000:1.0:1787763106.116762:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763106.116767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763106.116779:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763106.116787:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763106.116790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274246144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763106.116886:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763106.116945:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763106/real 1787763106] req@ffff9ac447840b40 x1874603274246016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763106.116980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274246016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763106.116990:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763106.116994:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763106.116998:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763106.117023:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246144:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763106.117042:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763106.117052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763106/real 1787763106] req@ffff9ac447842940 x1874603274246144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763106.117061:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274246144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763106.117067:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763106.117069:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763106.117071:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246144:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763106.692668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763107.140605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763107.716652:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763108.164650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763108.740668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763109.188613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763109.764675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763110.212624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763110.788651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763111.236595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763111.236601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763111.236609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763111.236614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763111.236636:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2155 00000100:00080000:0.0:1787763111.236651:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763111.236659:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763111.236714:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274246272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763111.236738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763111.236741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763111.236743:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763111.236746:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2155 00000100:00080000:0.0:1787763111.236749:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763111.236753:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763111.236765:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763111.236769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763111.236777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274246400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763111.236850:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763111.236901:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763111/real 1787763111] req@ffff9ac451f84b40 x1874603274246272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763111.236925:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274246272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763111.236935:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763111.236938:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763111.236942:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763111.236965:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246400:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763111.236983:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763111.237014:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763111/real 1787763111] req@ffff9ac451f87c00 x1874603274246400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763111.237029:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274246400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763111.237038:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763111.237042:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763111.237046:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246400:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763111.812692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763112.260621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763112.836652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763113.284600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763113.860700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763114.308602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763114.884658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763115.332594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763115.908705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763116.356686:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763116.356693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763116.356710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763116.356718:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763116.356748:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2161 00000100:00080000:0.0:1787763116.356768:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763116.356782:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763116.356845:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274246528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763116.356912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763116.356918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763116.356920:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763116.356927:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2161 00000100:00080000:0.0:1787763116.356932:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763116.356941:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763116.356956:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763116.356969:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763116.356981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274246656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763116.357073:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763116.357160:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763116/real 1787763116] req@ffff9ac451f84000 x1874603274246528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763116.357214:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274246528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763116.357247:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763116.357254:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763116.357260:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763116.357337:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246656:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763116.357377:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763116.357398:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763116/real 1787763116] req@ffff9ac451f870c0 x1874603274246656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763116.357419:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274246656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763116.357433:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763116.357439:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763116.357445:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246656:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763116.932685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763117.380599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763117.956686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763118.404625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763118.980656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763119.428599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763120.004741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763120.452623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763121.028723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763121.476614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763121.476676:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763121.476683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763121.476717:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763121.476728:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2166 00000100:00080000:3.0:1787763121.476741:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763121.476750:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763121.476795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274246784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763121.476825:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763121.476828:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763121.476830:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763121.476834:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2166 00000100:00080000:3.0:1787763121.476836:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763121.476839:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763121.476851:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763121.476862:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274246912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763121.476871:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246784:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763121.476954:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763121.477007:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763121/real 1787763121] req@ffff9ac451c52940 x1874603274246784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763121.477032:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274246784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763121.477042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763121.477046:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763121.477049:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274246784:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763121.477076:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246912:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763121.477097:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763121.477107:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763121/real 1787763121] req@ffff9ac451c50b40 x1874603274246912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763121.477116:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274246912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763121.477122:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763121.477125:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763121.477127:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274246912:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763122.052681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763122.500600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763123.076641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763123.524609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763124.100650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763124.548658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763125.124670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763125.572589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763126.148658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763126.596612:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763126.596617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763126.596625:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763126.596631:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763126.596642:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2171 00000100:00080000:3.0:1787763126.596656:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763126.596665:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763126.596716:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274247040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763126.596764:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763126.596767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763126.596769:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763126.596773:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2171 00000100:00080000:3.0:1787763126.596776:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763126.596780:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763126.596791:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763126.596798:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247040:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763126.596804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274247168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763126.596877:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763126.596919:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763126/real 1787763126] req@ffff9ac451c50780 x1874603274247040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763126.596945:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274247040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763126.596955:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763126.596958:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763126.596962:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247040:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763126.596987:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247168:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763126.597005:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763126.597015:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763126/real 1787763126] req@ffff9ac451c503c0 x1874603274247168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763126.597024:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274247168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763126.597030:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763126.597032:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763126.597034:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247168:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763127.172646:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763127.620594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763128.196730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763128.644614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763129.220748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763129.668655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763130.244776:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763130.692614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763131.268763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763131.716632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763131.716635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763131.716645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763131.716651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763131.716662:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2176 00000100:00080000:0.0:1787763131.716675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763131.716686:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763131.716741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274247296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763131.716772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763131.716775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763131.716776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763131.716780:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2176 00000100:00080000:0.0:1787763131.716783:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763131.716786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763131.716797:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763131.716808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274247424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763131.716820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763131.716978:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247424:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763132.292642:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763132.740624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763133.316757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763133.764650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763134.340722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763134.788606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763134.788923:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763134.788980:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763134.789001:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763134.789064:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787763134.789252:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763131/real 1787763134] req@ffff9ac451f861c0 x1874603274247424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763134.789315:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274247424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763134.789339:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763134.789348:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763134.789357:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247424:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787763134.789373:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763131/real 1787763134] req@ffff9ac451f84780 x1874603274247296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763134.789400:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274247296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763134.789417:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763134.789424:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763134.789431:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247296:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763135.364791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763135.812644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763136.388752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763136.836700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763136.836712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787763136.836719:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763136.836737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763136.836753:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2181 00000100:00080000:0.0:1787763136.836768:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763136.836776:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763136.836819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274247552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763136.836848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763136.836851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763136.836852:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763136.836855:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2181 00000100:00080000:0.0:1787763136.836857:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763136.836860:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763136.836870:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763136.836880:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274247680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763136.836919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247552:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763136.837007:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763136.837084:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763136/real 1787763136] req@ffff9ac451f85680 x1874603274247552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763136.837177:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274247552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763136.837196:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763136.837205:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763136.837212:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247552:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763136.837226:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247680:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763136.837276:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763136.837303:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763136/real 1787763136] req@ffff9ac451f843c0 x1874603274247680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763136.837327:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274247680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763136.837344:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763136.837350:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763136.837357:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247680:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763137.412669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763137.860596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763138.436666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763138.884578:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763139.460873:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763139.908726:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763140.484761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763140.932609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763141.508670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763141.956657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763141.956663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763141.956673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763141.956696:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763141.956710:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2186 00000100:00080000:0.0:1787763141.956727:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763141.956739:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763141.956818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274247808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763141.956854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763141.956881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763141.956885:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763141.956892:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2186 00000100:00080000:0.0:1787763141.956900:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763141.956907:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763141.956909:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247808:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763141.956923:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763141.956944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274247936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763141.957010:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763141.957084:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763141/real 1787763141] req@ffff9ac451f843c0 x1874603274247808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763141.957146:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274247808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763141.957166:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763141.957172:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763141.957177:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274247808:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763141.957209:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247936:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763141.957240:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763141.957256:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763141/real 1787763141] req@ffff9ac451f85680 x1874603274247936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763141.957270:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274247936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763141.957279:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763141.957282:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763141.957286:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274247936:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763142.532680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763142.980621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763143.556745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763144.004606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763144.580667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763145.028612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763145.604724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763146.052667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763146.628643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763147.076609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763147.076635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763147.076642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763147.076667:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763147.076679:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2191 00000100:00080000:1.0:1787763147.076693:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763147.076702:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763147.076747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274248064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763147.076773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763147.076777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763147.076780:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763147.076784:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2191 00000100:00080000:1.0:1787763147.076789:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763147.076794:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763147.076806:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763147.076807:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763147.076819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274248192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763147.076914:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763147.076969:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763147/real 1787763147] req@ffff9ac447843c00 x1874603274248064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763147.076992:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274248064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763147.077001:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763147.077004:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763147.077007:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763147.077030:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248192:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763147.077070:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763147.077082:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763147/real 1787763147] req@ffff9ac447842940 x1874603274248192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763147.077091:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274248192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763147.077097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763147.077099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763147.077102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248192:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763147.652635:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763148.100595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763148.676694:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763149.124635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763149.700711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763150.148621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763150.724712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763151.172663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763151.748690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763152.196626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763152.196673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763152.196683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763152.196688:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763152.196701:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2196 00000100:00080000:3.0:1787763152.196716:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763152.196726:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763152.196771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274248320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763152.196813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763152.196817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763152.196819:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763152.196824:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2196 00000100:00080000:3.0:1787763152.196829:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763152.196833:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763152.196847:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763152.196850:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763152.196873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274248448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763152.196938:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763152.196988:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763152/real 1787763152] req@ffff9ac451c50f00 x1874603274248320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763152.197032:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274248320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763152.197042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763152.197046:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763152.197049:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763152.197077:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248448:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763152.197099:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763152.197109:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763152/real 1787763152] req@ffff9ac451c52580 x1874603274248448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763152.197118:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274248448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763152.197124:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763152.197126:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763152.197129:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248448:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763152.772705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763153.220633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763153.796729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763154.244590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763154.820657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763155.268628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763155.844648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763156.292602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763156.868668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763157.316629:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763157.316641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763157.316643:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763157.316651:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763157.316665:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2202 00000100:00080000:1.0:1787763157.316680:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763157.316690:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763157.316741:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274248576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763157.316783:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763157.316809:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763157.316812:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763157.316818:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2202 00000100:00080000:1.0:1787763157.316823:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763157.316828:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248576:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763157.316834:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763157.316852:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763157.316872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274248704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763157.316927:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763157.316976:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763157/real 1787763157] req@ffff9ac447840b40 x1874603274248576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763157.317031:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274248576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763157.317059:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763157.317063:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763157.317067:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763157.317093:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248704:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763157.317135:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763157.317147:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763157/real 1787763157] req@ffff9ac447842d00 x1874603274248704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763157.317158:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274248704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763157.317166:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763157.317169:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763157.317172:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248704:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763157.892730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763158.340730:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763158.916707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763159.364635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763159.940664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763160.388600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763160.964791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763161.412635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763161.988668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763162.436608:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763162.436608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763162.436620:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763162.436626:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763162.436636:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2207 00000100:00080000:1.0:1787763162.436649:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763162.436656:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763162.436705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274248832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763162.436735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763162.436739:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763162.436741:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763162.436747:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2207 00000100:00080000:1.0:1787763162.436752:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763162.436757:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763162.436767:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248832:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763162.436771:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763162.436784:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274248960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763162.436849:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763162.436904:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763162/real 1787763162] req@ffff9ac4478430c0 x1874603274248832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763162.436933:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274248832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763162.436942:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763162.436946:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763162.436962:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274248832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763162.437000:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248960:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763162.437022:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763162.437033:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763162/real 1787763162] req@ffff9ac4478421c0 x1874603274248960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763162.437042:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274248960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763162.437048:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763162.437051:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763162.437053:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274248960:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763163.012651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763163.460614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763164.036680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763164.484634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763165.060769:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763165.508598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763166.084662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763166.532611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763167.108679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763167.556621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763167.556628:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763167.556637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763167.556643:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763167.556657:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2212 00000100:00080000:0.0:1787763167.556673:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763167.556682:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763167.556736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274249088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763167.556800:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763167.556805:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763167.556808:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763167.556815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2212 00000100:00080000:0.0:1787763167.556820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763167.556837:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763167.556840:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249088:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763167.556852:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763167.556872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274249216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763167.556933:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763167.557067:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763167/real 1787763167] req@ffff9ac451f84780 x1874603274249088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763167.557096:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274249088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763167.557119:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763167.557125:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763167.557129:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763167.557162:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249216:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763167.557191:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763167.557206:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763167/real 1787763167] req@ffff9ac451f861c0 x1874603274249216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763167.557219:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274249216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763167.557228:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763167.557232:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763167.557235:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249216:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763168.132659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763168.580605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763169.156650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763169.604636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763170.180658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763170.628607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763171.204666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763171.652607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763172.228693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763172.676624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763172.676668:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763172.676692:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763172.676697:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763172.676721:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2217 00000100:00080000:1.0:1787763172.676745:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763172.676755:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763172.676809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274249344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763172.676839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763172.676843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763172.676845:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763172.676849:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2217 00000100:00080000:1.0:1787763172.676853:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763172.676857:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763172.676870:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763172.676895:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763172.676901:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274249472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763172.677029:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763172.677090:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763172/real 1787763172] req@ffff9ac4478403c0 x1874603274249344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763172.677140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274249344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763172.677151:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763172.677155:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763172.677169:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763172.677196:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249472:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763172.677219:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763172.677231:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763172/real 1787763172] req@ffff9ac447841680 x1874603274249472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763172.677241:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274249472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763172.677249:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763172.677251:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763172.677254:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763173.252686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763173.700617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763174.276689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763174.724626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763175.300686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763175.748662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763176.324671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763176.772678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763177.348681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763177.796603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763177.796647:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763177.796654:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763177.796659:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763177.796669:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2222 00000100:00080000:0.0:1787763177.796682:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763177.796690:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763177.796736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274249600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763177.796764:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763177.796767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763177.796787:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763177.796793:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249600:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787763177.796794:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2222 00000100:00080000:0.0:1787763177.796801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763177.796805:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763177.796817:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763177.796829:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274249728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763177.796895:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763177.796948:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763177/real 1787763177] req@ffff9ac451f870c0 x1874603274249600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763177.796971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274249600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763177.796981:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763177.796984:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763177.796987:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763177.797012:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249728:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763177.797031:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763177.797041:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763177/real 1787763177] req@ffff9ac451f84000 x1874603274249728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763177.797049:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274249728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763177.797055:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763177.797058:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763177.797060:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249728:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763178.372702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763178.820633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763179.396649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763179.844603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763180.420717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763180.868642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763181.444730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763181.892607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763182.468691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763182.916644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763182.916678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763182.916690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763182.916697:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763182.916713:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2227 00000100:00080000:3.0:1787763182.916732:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763182.916744:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763182.916803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274249856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763182.916853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763182.916873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763182.916875:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763182.916879:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2227 00000100:00080000:3.0:1787763182.916883:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763182.916888:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763182.916899:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763182.916906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763182.916916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274249984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763182.916992:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763182.917051:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763182/real 1787763182] req@ffff9ac451c52580 x1874603274249856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763182.917088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274249856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763182.917098:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763182.917102:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763182.917106:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274249856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763182.917133:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249984:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763182.917155:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763182.917166:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763182/real 1787763182] req@ffff9ac451c50f00 x1874603274249984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763182.917176:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274249984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763182.917183:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763182.917186:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763182.917188:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274249984:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763183.492761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763183.940630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763184.516697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763184.964611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763185.540658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763185.988618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763186.564659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763187.012601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763187.588679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763188.036633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763188.036636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763188.036660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763188.036665:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763188.036676:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2232 00000100:00080000:3.0:1787763188.036689:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763188.036698:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763188.036740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274250112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763188.036768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763188.036771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763188.036773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763188.036788:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2232 00000100:00080000:3.0:1787763188.036791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763188.036795:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763188.036806:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763188.036820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274250240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763188.036826:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250112:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763188.036908:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763188.036955:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763188/real 1787763188] req@ffff9ac451c503c0 x1874603274250112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763188.036982:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274250112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763188.036992:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763188.036996:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763188.037000:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250112:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763188.037025:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274250240:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763188.037045:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763188.037054:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763188/real 1787763188] req@ffff9ac451c50780 x1874603274250240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763188.037063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274250240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763188.037068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763188.037071:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763188.037073:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274250240:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763188.612709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763189.060603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763189.636646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763190.084647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763190.660734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763191.108647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763191.684780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763192.132615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763192.708715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763193.156613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763193.156686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763193.156698:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763193.156703:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763193.156717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2237 00000100:00080000:0.0:1787763193.156731:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763193.156764:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763193.156812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274250368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763193.156868:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763193.156874:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763193.156876:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763193.156882:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2237 00000100:00080000:0.0:1787763193.156911:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763193.156919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250368:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787763193.156924:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763193.156947:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763193.156973:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274250496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763193.157031:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763193.157092:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763193/real 1787763193] req@ffff9ac451f87c00 x1874603274250368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763193.157132:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274250368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763193.157143:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763193.157147:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763193.157151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763193.157179:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274250496:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763193.157200:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763193.157211:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763193/real 1787763193] req@ffff9ac451f84b40 x1874603274250496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763193.157220:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274250496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763193.157227:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763193.157230:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763193.157232:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274250496:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763193.732800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763194.180599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763194.756700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763195.204630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763195.780753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763196.228628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763196.804674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763197.252586:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763197.828656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763198.276670:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763198.276680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763198.276684:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763198.276691:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763198.276725:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2243 00000100:00080000:0.0:1787763198.276742:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763198.276772:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763198.276823:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274250624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763198.276881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763198.276886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763198.276889:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763198.276894:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2243 00000100:00080000:0.0:1787763198.276899:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763198.276904:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763198.276916:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763198.276951:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274250752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763198.276955:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763198.277161:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274250752:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763198.852668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763199.300612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763199.876763:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763200.324640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763200.900664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763201.348597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763201.348851:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763201.348898:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763201.348920:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763201.348965:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787763201.349003:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763198/real 1787763201] req@ffff9ac451f87c00 x1874603274250752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763201.349035:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274250752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763201.349048:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763201.349054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763201.349059:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274250752:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787763201.349068:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763198/real 1787763201] req@ffff9ac451f84b40 x1874603274250624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763201.349081:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274250624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763201.349089:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763201.349093:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763201.349103:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250624:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787763201.924781:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763202.372634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763202.948709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763203.396659:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763203.396685:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787763203.396689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763203.396690:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763203.396722:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2248 00000100:00080000:0.0:1787763203.396738:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763203.396746:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763203.396790:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274250880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763203.396817:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763203.396821:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763203.396822:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763203.396826:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2248 00000100:00080000:0.0:1787763203.396829:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763203.396833:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763203.396845:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763203.396856:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763203.396860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274251008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763203.396941:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763203.397001:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763203/real 1787763203] req@ffff9ac451f84000 x1874603274250880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763203.397042:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274250880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763203.397051:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763203.397055:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763203.397058:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274250880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763203.397088:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251008:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763203.397115:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763203.397137:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763203/real 1787763203] req@ffff9ac451f870c0 x1874603274251008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763203.397151:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274251008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763203.397161:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763203.397164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763203.397168:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251008:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763203.972682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763204.420613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763204.996771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763205.444615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763206.020756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763206.468624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763207.044741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763207.492600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763208.068908:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763208.516689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763208.516754:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763208.516764:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763208.516794:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763208.516807:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2253 00000100:00080000:1.0:1787763208.516824:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763208.516834:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763208.516884:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274251136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763208.516926:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763208.516931:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763208.516934:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763208.516940:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2253 00000100:00080000:1.0:1787763208.516944:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763208.516950:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763208.516964:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763208.516985:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274251264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763208.516988:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251136:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763208.517111:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763208.517168:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763208/real 1787763208] req@ffff9ac447840780 x1874603274251136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763208.517206:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274251136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763208.517222:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763208.517229:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763208.517237:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763208.517281:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251264:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763208.517330:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763208.517357:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763208/real 1787763208] req@ffff9ac447840f00 x1874603274251264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763208.517374:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274251264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763208.517386:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763208.517390:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763208.517395:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251264:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763209.092802:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763209.540694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763210.116684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763210.564590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763211.140772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763211.588625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763212.164700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763212.612724:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763213.188760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763213.636673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763213.636688:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763213.636699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763213.636705:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763213.636720:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2258 00000100:00080000:1.0:1787763213.636734:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763213.636744:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763213.636793:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274251392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763213.636831:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763213.636851:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763213.636854:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763213.636860:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2258 00000100:00080000:1.0:1787763213.636866:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763213.636873:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763213.636887:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763213.636909:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763213.636913:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274251520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763213.637006:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763213.637074:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763213/real 1787763213] req@ffff9ac447840000 x1874603274251392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763213.637144:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274251392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763213.637165:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763213.637174:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763213.637182:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763213.637222:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251520:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763213.637268:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763213.637288:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763213/real 1787763213] req@ffff9ac447842580 x1874603274251520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763213.637305:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274251520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763213.637317:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763213.637321:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763213.637325:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251520:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763214.212710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763214.660667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763215.236675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763215.684684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763216.260773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763216.708731:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763217.284813:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763217.732644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763218.308713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763218.756675:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763218.756690:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763218.756696:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763218.756711:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2263 00000100:00080000:2.0:1787763218.756729:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763218.756740:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763218.756819:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274251648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763218.756865:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763218.756871:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763218.756875:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763218.756882:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2263 00000100:00080000:2.0:1787763218.756888:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763218.756894:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763218.756909:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763218.756922:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763218.756934:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274251776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763218.756990:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000800:00000100:1.0:1787763218.757008:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763218.757070:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763218/real 1787763218] req@ffff9ac4523bf840 x1874603274251648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763218.757085:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274251648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763218.757095:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763218.757098:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763218.757102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763218.757124:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251776:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763218.757141:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763218.757152:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763218/real 1787763218] req@ffff9ac4523bda40 x1874603274251776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763218.757167:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274251776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763218.757190:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763218.757195:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763218.757199:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274251776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763219.332744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763219.780644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763220.356731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763220.804665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763221.380684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763221.828626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763222.404645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763222.852646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763223.428681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763223.876626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763223.876633:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763223.876643:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763223.876650:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763223.876665:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2268 00000100:00080000:2.0:1787763223.876679:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763223.876689:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763223.876737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274251904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763223.876769:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763223.876799:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787763223.876802:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251904:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787763223.876803:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763223.876816:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2268 00000100:00080000:2.0:1787763223.876822:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763223.876829:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763223.876844:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763223.876862:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274252032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763223.876886:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763223.876931:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763223/real 1787763223] req@ffff9ac4523bda40 x1874603274251904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763223.876952:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274251904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763223.876965:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763223.876971:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763223.876976:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274251904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763223.877009:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252032:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763223.877034:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763223.877050:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763223/real 1787763223] req@ffff9ac4523bf840 x1874603274252032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763223.877066:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274252032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763223.877086:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763223.877089:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763223.877091:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763224.452670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763224.900614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763225.476669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763225.924643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763226.500655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763226.948639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763227.524748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763227.972647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763228.548634:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763228.996641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763228.996651:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763228.996659:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763228.996663:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763228.996674:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2273 00000100:00080000:0.0:1787763228.996687:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763228.996694:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763228.996738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274252160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763228.996770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763228.996774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763228.996796:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763228.996804:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2273 00000100:00080000:0.0:1787763228.996811:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763228.996816:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763228.996820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252160:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763228.996827:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763228.996849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274252288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763228.996903:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763228.996938:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763228/real 1787763228] req@ffff9ac451f861c0 x1874603274252160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763228.996954:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274252160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763228.996963:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763228.996966:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763228.996970:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763228.996989:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252288:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763228.997005:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763228.997013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763228/real 1787763228] req@ffff9ac451f84780 x1874603274252288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763228.997022:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274252288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763228.997027:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763228.997029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763228.997032:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252288:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763229.572668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763230.020602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763230.596686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763231.044770:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763231.620672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763232.068606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763232.644672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763233.092603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763233.668641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763234.116617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763234.116620:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763234.116628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763234.116633:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763234.116644:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2278 00000100:00080000:3.0:1787763234.116659:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763234.116667:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763234.116715:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274252416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763234.116743:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763234.116746:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763234.116747:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763234.116751:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2278 00000100:00080000:3.0:1787763234.116753:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763234.116756:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763234.116766:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763234.116799:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763234.116800:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274252544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763234.116879:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763234.116931:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763234/real 1787763234] req@ffff9ac451c50b40 x1874603274252416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763234.116962:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274252416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763234.116972:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763234.116976:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763234.116979:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763234.117004:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252544:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763234.117025:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763234.117035:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763234/real 1787763234] req@ffff9ac451c52940 x1874603274252544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763234.117044:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274252544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763234.117050:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763234.117052:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763234.117054:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763234.692693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763235.140598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763235.716679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763236.164587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763236.740676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763237.188624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763237.764756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763238.212630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763238.788798:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763239.236620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763239.236647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763239.236651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763239.236655:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763239.236664:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2283 00000100:00080000:0.0:1787763239.236675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763239.236685:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763239.236723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274252672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763239.236768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763239.236772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763239.236774:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763239.236779:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2283 00000100:00080000:0.0:1787763239.236784:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763239.236788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763239.236799:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763239.236813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274252800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787763239.236847:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252672:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787763239.236902:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763239.236933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763239/real 1787763239] req@ffff9ac451f85680 x1874603274252672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763239.236945:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274252672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763239.236964:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763239.236967:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763239.236970:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763239.236975:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252800:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763239.236991:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763239.236999:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763239/real 1787763239] req@ffff9ac451f843c0 x1874603274252800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763239.237007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274252800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763239.237014:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763239.237017:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763239.237021:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274252800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763239.812760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763240.260667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763240.836771:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763241.284604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763241.860714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763242.308598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763242.884658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763243.332607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763243.908643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763244.356617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763244.356629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763244.356634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763244.356646:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2289 00000100:00100000:2.0:1787763244.356668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763244.356672:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763244.356683:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763244.356743:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274252928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763244.356765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763244.356790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763244.356791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763244.356795:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2289 00000100:00080000:0.0:1787763244.356798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763244.356802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763244.356811:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763244.356823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274253056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787763244.356848:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252928:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787763244.356920:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763244.356960:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763244/real 1787763244] req@ffff9ac451f843c0 x1874603274252928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763244.356974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274252928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763244.356983:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763244.356986:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763244.356989:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274252928:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763244.357002:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253056:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763244.357019:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763244.357028:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763244/real 1787763244] req@ffff9ac451f85680 x1874603274253056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763244.357037:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274253056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763244.357042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763244.357045:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763244.357047:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253056:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763244.932695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763245.380634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763245.956805:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763246.404709:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763246.980792:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763247.428724:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763248.004678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763248.452742:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763249.028731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763249.476616:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763249.476628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763249.476634:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763249.476646:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2294 00000100:00080000:1.0:1787763249.476661:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763249.476669:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763249.476692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787763249.476727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274253184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763249.476774:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763249.476777:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763249.476779:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763249.476783:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2294 00000100:00080000:1.0:1787763249.476787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763249.476807:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763249.476819:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763249.476838:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763249.476839:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274253312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763249.476964:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763249.477020:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763249/real 1787763249] req@ffff9ac447843840 x1874603274253184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763249.477043:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274253184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763249.477070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763249.477076:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763249.477081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763249.477133:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253312:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763249.477168:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763249.477185:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763249/real 1787763249] req@ffff9ac447841a40 x1874603274253312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763249.477202:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274253312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763249.477213:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763249.477218:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763249.477222:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763250.052805:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763250.500681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763251.076737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763251.524723:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763252.100792:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763252.548692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763253.124768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763253.572732:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763254.148788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763254.596607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763254.596609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763254.596619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763254.596624:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763254.596636:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2299 00000100:00080000:1.0:1787763254.596649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763254.596657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763254.596704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274253440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763254.596732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763254.596735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763254.596737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763254.596741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2299 00000100:00080000:1.0:1787763254.596743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763254.596746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763254.596756:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763254.596765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274253568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763254.596935:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253440:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763254.597033:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763254.597093:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763254/real 1787763254] req@ffff9ac447841e00 x1874603274253440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763254.597118:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274253440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763254.597132:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763254.597138:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763254.597143:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763254.597152:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253568:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763254.597209:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763254.597224:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763254/real 1787763254] req@ffff9ac4478412c0 x1874603274253568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763254.597240:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274253568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763254.597251:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763254.597255:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763254.597259:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253568:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763255.172776:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763255.620599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763256.196716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763256.644593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763257.220812:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763257.668615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763258.244709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763258.692617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763259.268759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763259.716650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763259.716663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763259.716670:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763259.716673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763259.716685:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2304 00000100:00080000:3.0:1787763259.716700:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763259.716712:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763259.716755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274253696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763259.716785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763259.716789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763259.716791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763259.716796:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2304 00000100:00080000:3.0:1787763259.716801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763259.716820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253696:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787763259.716828:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763259.716844:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763259.716869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274253824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763259.716896:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763259.716973:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763259/real 1787763259] req@ffff9ac451c530c0 x1874603274253696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763259.716997:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274253696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763259.717011:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763259.717017:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763259.717023:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253696:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763259.717055:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253824:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763259.717086:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763259.717100:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763259/real 1787763259] req@ffff9ac451c52d00 x1874603274253824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763259.717149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274253824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763259.717158:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763259.717162:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763259.717166:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274253824:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763260.292673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763260.740600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763261.316741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763261.764634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763262.340766:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763262.788645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763263.364754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763263.812653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763264.388641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763264.836607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763264.836617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763264.836622:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763264.836632:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2309 00000100:00080000:3.0:1787763264.836649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787763264.836655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763264.836663:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763264.836715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274253952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763264.836749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763264.836753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763264.836756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763264.836761:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2309 00000100:00080000:3.0:1787763264.836768:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763264.836773:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763264.836787:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763264.836807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274254080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763264.836811:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763264.836949:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254080:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763265.412773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763265.860641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763266.436801:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763266.884673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763267.460696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763267.908699:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787763267.909027:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787763267.909107:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787763267.909126:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787763267.909174:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787763267.909256:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763264/real 1787763267] req@ffff9ac451c530c0 x1874603274254080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763267.909348:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274254080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763267.909368:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763267.909375:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763267.909382:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254080:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787763267.909395:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763264/real 1787763267] req@ffff9ac451c52d00 x1874603274253952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763267.909415:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274253952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763267.909427:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763267.909432:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763267.909438:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274253952:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763268.484750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763268.932702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763269.508755:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763269.956699:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763269.956717:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763269.956724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763269.956742:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2314 00000100:00080000:2.0:1787763269.956759:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763269.956772:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763269.956784:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787763269.956847:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274254208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763269.956932:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763269.956939:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763269.956952:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763269.956962:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2314 00000100:00080000:2.0:1787763269.956972:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763269.956992:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763269.957011:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763269.957035:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763269.957047:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274254336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787763269.957175:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763269.957278:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763269/real 1787763269] req@ffff9ac4523bc3c0 x1874603274254208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763269.957316:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274254208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763269.957338:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763269.957347:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763269.957355:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763269.957414:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254336:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787763269.957466:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787763269.957591:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763269/real 1787763269] req@ffff9ac4523be940 x1874603274254336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787763269.957623:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274254336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787763269.957636:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787763269.957643:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787763269.957650:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254336:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763270.532720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763270.980801:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763271.556824:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763272.004643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763272.580736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763273.028760:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763273.604710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763274.052651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763274.628785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763275.076698:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763275.076702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763275.076716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763275.076726:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763275.076739:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2319 00000100:00080000:1.0:1787763275.076752:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763275.076762:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763275.076820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274254464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763275.076857:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763275.076862:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763275.076864:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763275.076870:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2319 00000100:00080000:1.0:1787763275.076875:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763275.076897:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763275.076911:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763275.076919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763275.076936:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274254592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763275.077006:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763275.077052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763275/real 1787763275] req@ffff9ac447843480 x1874603274254464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763275.077073:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274254464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763275.077087:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763275.077091:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763275.077096:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763275.077123:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254592:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763275.077148:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763275.077199:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763275/real 1787763275] req@ffff9ac4478412c0 x1874603274254592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763275.077218:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274254592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763275.077229:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763275.077233:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763275.077237:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254592:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763275.652717:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763276.100700:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763276.676728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763277.124647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763277.700701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763278.148661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763278.724763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763279.172644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763279.748773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763280.196660:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763280.196670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763280.196674:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763280.196681:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763280.196696:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2324 00000100:00080000:1.0:1787763280.196712:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763280.196722:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763280.196769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274254720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763280.196808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763280.196813:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763280.196816:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763280.196825:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2324 00000100:00080000:1.0:1787763280.196832:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763280.196838:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763280.196851:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763280.196858:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763280.196872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274254848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763280.196970:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763280.197017:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763280/real 1787763280] req@ffff9ac447841e00 x1874603274254720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763280.197032:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274254720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763280.197042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763280.197046:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763280.197050:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763280.197087:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254848:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763280.197104:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763280.197134:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763280/real 1787763280] req@ffff9ac447841a40 x1874603274254848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763280.197162:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274254848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763280.197169:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763280.197183:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763280.197185:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274254848:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763280.772788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763281.220712:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763281.796769:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763282.244630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763282.820716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763283.268608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763283.844788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763284.292622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763284.868755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763285.316628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763285.316632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763285.316641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763285.316645:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763285.316656:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2330 00000100:00080000:3.0:1787763285.316667:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763285.316674:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763285.316705:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274254976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763285.316728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763285.316731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763285.316732:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763285.316736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2330 00000100:00080000:3.0:1787763285.316738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763285.316740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763285.316748:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763285.316772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274255104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763285.316813:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254976:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763285.316870:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763285.316902:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763285/real 1787763285] req@ffff9ac451c52940 x1874603274254976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763285.316916:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274254976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763285.316924:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763285.316927:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763285.316930:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274254976:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763285.316935:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255104:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763285.316949:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763285.316956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763285/real 1787763285] req@ffff9ac451c50b40 x1874603274255104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763285.316965:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274255104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763285.316982:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763285.316985:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763285.316987:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255104:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763285.892706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763286.340634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763286.916772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763287.364594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763287.940701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763288.388678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763288.964695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763289.412613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763289.988648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763290.436603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763290.436612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763290.436614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763290.436621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763290.436633:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2335 00000100:00080000:3.0:1787763290.436646:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763290.436655:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763290.436710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274255232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763290.436739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763290.436743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763290.436744:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763290.436748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2335 00000100:00080000:3.0:1787763290.436751:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763290.436754:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763290.436766:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763290.436783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274255360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763290.436792:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274255232:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763290.436873:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763290.436917:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763290/real 1787763290] req@ffff9ac451c50780 x1874603274255232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763290.436932:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274255232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763290.436941:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763290.436945:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763290.436948:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274255232:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763290.436973:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255360:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763290.436989:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763290.436998:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763290/real 1787763290] req@ffff9ac451c503c0 x1874603274255360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763290.437006:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274255360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763290.437011:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763290.437013:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763290.437015:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255360:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763291.012683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763291.460615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763292.036656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763292.484610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763293.060670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763293.508603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763294.084725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763294.532606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763295.108654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763295.556670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763295.556701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763295.556708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763295.556733:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763295.556747:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2340 00000100:00080000:1.0:1787763295.556763:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763295.556774:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763295.556830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274255488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763295.556872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763295.556875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763295.556876:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763295.556880:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2340 00000100:00080000:1.0:1787763295.556883:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763295.556887:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763295.556898:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763295.556910:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274255616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763295.556916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274255488:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763295.557006:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763295.557276:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763295/real 1787763295] req@ffff9ac447843840 x1874603274255488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763295.557295:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274255488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763295.557306:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763295.557311:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763295.557316:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274255488:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763295.557342:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255616:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763295.557363:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763295.557374:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763295/real 1787763295] req@ffff9ac447842580 x1874603274255616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763295.557383:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274255616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763295.557389:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763295.557391:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763295.557394:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255616:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763296.132651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763296.580616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763297.156699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763297.604620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763298.180676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763298.628603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763299.204635:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763299.652633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763300.228647:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763300.676634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763300.676691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763300.676703:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763300.676709:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763300.676721:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2345 00000100:00080000:1.0:1787763300.676735:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763300.676745:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763300.676799:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274255744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763300.676853:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763300.676883:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763300.676886:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763300.676893:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2345 00000100:00080000:1.0:1787763300.676900:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763300.676907:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763300.676922:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763300.676950:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274255872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763300.676954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274255744:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763300.677104:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763300.677159:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763300/real 1787763300] req@ffff9ac447840000 x1874603274255744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763300.677208:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274255744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763300.677227:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763300.677234:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763300.677241:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274255744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763300.677281:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255872:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763300.677311:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763300.677326:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763300/real 1787763300] req@ffff9ac447840f00 x1874603274255872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763300.677339:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274255872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763300.677347:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763300.677351:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763300.677354:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274255872:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763301.252774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763301.700611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763302.276776:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763302.724631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763303.300789:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763303.748667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763304.324780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763304.772659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763305.348798:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763305.796619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763305.796630:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763305.796636:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763305.796648:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2350 00000100:00100000:0.0:1787763305.796651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763305.796663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763305.796674:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763305.796718:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274256000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763305.796759:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763305.796764:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763305.796766:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763305.796771:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2350 00000100:00080000:1.0:1787763305.796775:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763305.796779:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763305.796789:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763305.796802:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274256128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763305.796874:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256000:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763305.796950:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763305.797025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763305/real 1787763305] req@ffff9ac447840780 x1874603274256000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763305.797068:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274256000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763305.797080:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763305.797085:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763305.797089:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256000:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763305.797098:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256128:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763305.797126:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763305.797140:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763305/real 1787763305] req@ffff9ac447841680 x1874603274256128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763305.797153:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274256128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763305.797162:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763305.797166:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763305.797169:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256128:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763306.372746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763306.820640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763307.396771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763307.844652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763308.420805:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763308.868721:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763309.444773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763309.892684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763310.468792:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763310.916676:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763310.916683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763310.916693:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763310.916701:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763310.916713:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2355 00000100:00080000:1.0:1787763310.916727:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763310.916738:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763310.916788:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274256256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763310.916828:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763310.916833:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763310.916836:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763310.916841:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2355 00000100:00080000:1.0:1787763310.916846:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763310.916850:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763310.916862:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763310.916880:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274256384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763310.916905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256256:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763310.917025:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763310.917083:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763310/real 1787763310] req@ffff9ac4478403c0 x1874603274256256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763310.917107:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274256256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763310.917140:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763310.917146:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763310.917151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256256:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763310.917180:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256384:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763310.917213:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763310.917229:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763310/real 1787763310] req@ffff9ac4478421c0 x1874603274256384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763310.917244:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274256384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763310.917255:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763310.917259:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763310.917263:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256384:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763311.492785:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763311.940616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763312.516810:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763312.964612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763313.540725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763313.988660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763314.564793:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763315.012686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763315.588767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763316.036664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763316.036696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763316.036721:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763316.036729:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763316.036744:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2360 00000100:00080000:2.0:1787763316.036757:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763316.036778:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763316.036830:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274256512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763316.036879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763316.036887:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763316.036892:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763316.036921:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2360 00000100:00080000:2.0:1787763316.036931:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763316.036940:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763316.036954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256512:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787763316.036968:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763316.037007:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274256640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763316.037063:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763316.037119:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763316/real 1787763316] req@ffff9ac4523bc000 x1874603274256512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763316.037144:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274256512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763316.037198:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763316.037204:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763316.037209:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256512:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763316.037240:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256640:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763316.037275:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763316.037292:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763316/real 1787763316] req@ffff9ac4523bd680 x1874603274256640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763316.037308:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274256640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763316.037330:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763316.037334:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763316.037338:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256640:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763316.612798:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763317.060731:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763317.636637:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763318.084638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763318.660672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763319.108638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763319.684742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763320.132643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763320.708746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763321.156671:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763321.156689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763321.156697:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763321.156716:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2365 00000100:00100000:1.0:1787763321.156720:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763321.156762:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763321.156774:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763321.156820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274256768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763321.156881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763321.156889:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763321.156893:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763321.156899:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2365 00000100:00080000:2.0:1787763321.156905:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763321.156911:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763321.156925:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763321.156945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274256896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763321.156965:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256768:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763321.157055:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763321.157126:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763321/real 1787763321] req@ffff9ac4523bc780 x1874603274256768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763321.157152:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274256768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763321.157167:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763321.157184:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763321.157190:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274256768:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763321.157220:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256896:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763321.157254:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763321.157270:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763321/real 1787763321] req@ffff9ac4523be580 x1874603274256896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763321.157286:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274256896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763321.157296:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763321.157300:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763321.157305:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274256896:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763321.732763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763322.180646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763322.756741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763323.204623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763323.780697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763324.228708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763324.804709:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763325.252662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763325.828643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763326.276628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763326.276633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763326.276645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763326.276649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763326.276678:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2371 00000100:00080000:3.0:1787763326.276693:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763326.276702:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763326.276747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274257024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763326.276776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763326.276789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763326.276791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763326.276797:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2371 00000100:00080000:3.0:1787763326.276801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763326.276805:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763326.276815:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257024:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787763326.276817:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763326.276863:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274257152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763326.276947:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763326.276993:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763326/real 1787763326] req@ffff9ac451c50f00 x1874603274257024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763326.277009:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274257024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763326.277018:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763326.277022:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763326.277026:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257024:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763326.277076:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257152:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763326.277094:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763326.277104:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763326/real 1787763326] req@ffff9ac451c52580 x1874603274257152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763326.277112:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274257152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763326.277118:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763326.277121:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763326.277123:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257152:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763326.852664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763327.300619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763327.876771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763328.324606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763328.900679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763329.348619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763329.924672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763330.372634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763330.948707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763331.396627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763331.396643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763331.396649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763331.396651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763331.396666:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2376 00000100:00080000:3.0:1787763331.396682:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763331.396693:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763331.396744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274257280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763331.396777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763331.396782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763331.396784:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763331.396803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2376 00000100:00080000:3.0:1787763331.396808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763331.396814:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763331.396829:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257280:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787763331.396849:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763331.396870:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274257408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763331.396964:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257408:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763331.972673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763332.420734:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763332.996840:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763333.444603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763334.020676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763334.468647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763334.468862:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763334.468912:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763334.468932:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763334.468984:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787763334.469037:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763331/real 1787763334] req@ffff9ac451c50f00 x1874603274257408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763334.469063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274257408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763334.469075:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763334.469080:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763334.469088:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257408:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787763334.469113:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763331/real 1787763334] req@ffff9ac451c52580 x1874603274257280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763334.469124:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274257280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763334.469130:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763334.469133:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763334.469136:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257280:192.168.203.123@tcp:250:kworker.0 00010000:00010000:0.0:1787763335.044754:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763335.492686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763336.068687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763336.516682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763336.516699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763336.516707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763336.516726:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2381 00000100:00080000:3.0:1787763336.516746:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763336.516776:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763336.516798:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787763336.516844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274257536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763336.516883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763336.516887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763336.516889:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763336.516894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2381 00000100:00080000:3.0:1787763336.516899:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763336.516903:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763336.516917:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763336.516932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274257664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763336.516976:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257536:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763336.517082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763336.517147:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763336/real 1787763336] req@ffff9ac451c503c0 x1874603274257536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763336.517175:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274257536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763336.517189:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763336.517196:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763336.517202:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257536:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763336.517213:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257664:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763336.517250:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763336.517270:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763336/real 1787763336] req@ffff9ac451c50780 x1874603274257664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763336.517288:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274257664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763336.517325:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763336.517331:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763336.517336:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257664:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763337.092783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763337.540731:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763338.116802:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763338.564623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763339.141029:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763339.588642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763340.164723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763340.612671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763341.188798:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763341.636599:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763341.636611:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763341.636617:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763341.636631:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2386 00000100:00080000:3.0:1787763341.636646:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787763341.636652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763341.636657:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763341.636712:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274257792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763341.636756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763341.636759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763341.636761:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763341.636765:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2386 00000100:00080000:3.0:1787763341.636767:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763341.636771:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763341.636782:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763341.636794:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274257920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763341.636852:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257792:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763341.636968:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763341.637025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763341/real 1787763341] req@ffff9ac451c50b40 x1874603274257792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763341.637050:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274257792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763341.637064:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763341.637070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763341.637076:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274257792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763341.637086:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257920:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763341.637117:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763341.637133:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763341/real 1787763341] req@ffff9ac451c52940 x1874603274257920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763341.637148:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274257920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763341.637159:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763341.637164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763341.637168:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274257920:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763342.212781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763342.660620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763343.236695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763343.684687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763344.260754:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763344.708659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763345.284716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763345.732652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763346.308741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763346.756607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763346.756627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763346.756640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763346.756646:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763346.756658:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2391 00000100:00080000:3.0:1787763346.756707:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763346.756717:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763346.756752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274258048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763346.756782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763346.756785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763346.756787:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763346.756791:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2391 00000100:00080000:3.0:1787763346.756794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763346.756797:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763346.756806:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763346.756817:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274258176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763346.756852:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258048:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763346.756920:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763346.756959:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763346/real 1787763346] req@ffff9ac451c52d00 x1874603274258048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763346.756975:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274258048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763346.756985:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763346.756988:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763346.756992:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258048:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763346.756998:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258176:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763346.757016:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763346.757025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763346/real 1787763346] req@ffff9ac451c530c0 x1874603274258176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763346.757035:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274258176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763346.757042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763346.757045:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763346.757047:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258176:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763347.332692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763347.780634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763348.356739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763348.804602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763349.380680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763349.828607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763350.404695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763350.852622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763351.428734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763351.876594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763351.876600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763351.876608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763351.876614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763351.876626:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2396 00000100:00080000:3.0:1787763351.876638:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763351.876661:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763351.876727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274258304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763351.876759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763351.876782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763351.876785:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763351.876790:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2396 00000100:00080000:3.0:1787763351.876794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763351.876800:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258304:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787763351.876802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763351.876817:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763351.876836:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274258432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763351.876882:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763351.876924:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763351/real 1787763351] req@ffff9ac451c530c0 x1874603274258304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763351.876946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274258304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763351.876956:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763351.876960:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763351.876963:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258304:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763351.876992:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258432:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763351.877011:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763351.877020:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763351/real 1787763351] req@ffff9ac451c52d00 x1874603274258432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763351.877029:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274258432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763351.877037:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763351.877040:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763351.877044:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258432:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763352.452629:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763352.900689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763353.476874:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763353.924660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763354.500799:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763354.948601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763355.524707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763355.972616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763356.548673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763356.996636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763356.996662:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763356.996688:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763356.996696:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763356.996714:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2401 00000100:00080000:0.0:1787763356.996733:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763356.996747:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763356.996844:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274258560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763356.996878:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763356.996906:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763356.996909:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763356.996916:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2401 00000100:00080000:0.0:1787763356.996922:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763356.996930:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258560:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787763356.996932:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763356.996950:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763356.996968:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274258688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763356.997015:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763356.997060:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763356/real 1787763356] req@ffff9ac451f84780 x1874603274258560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763356.997075:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274258560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763356.997096:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763356.997099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763356.997103:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258560:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763356.997133:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258688:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763356.997160:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763356.997175:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763356/real 1787763356] req@ffff9ac451f861c0 x1874603274258688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763356.997184:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274258688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763356.997190:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763356.997193:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763356.997195:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258688:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763357.572703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763358.020635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763358.596675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763359.044621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763359.620702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763360.068621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763360.644684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763361.092629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763361.668719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763362.116643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763362.116703:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763362.116710:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763362.116716:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763362.116728:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2406 00000100:00080000:1.0:1787763362.116741:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763362.116751:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763362.116811:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274258816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763362.116848:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763362.116853:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763362.116856:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763362.116861:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2406 00000100:00080000:1.0:1787763362.116865:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763362.116870:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763362.116882:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763362.116898:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274258944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763362.116953:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258816:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763362.117061:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763362.117121:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763362/real 1787763362] req@ffff9ac4478430c0 x1874603274258816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763362.117144:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274258816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763362.117158:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763362.117164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763362.117169:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274258816:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763362.117204:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258944:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763362.117234:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763362.117249:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763362/real 1787763362] req@ffff9ac447842d00 x1874603274258944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763362.117262:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274258944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763362.117271:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763362.117275:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763362.117279:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274258944:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763362.692809:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763363.140647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763363.716703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763364.164674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763364.740651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763365.188657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763365.764670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763366.212647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763366.788664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763367.236687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763367.236712:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763367.236726:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763367.236731:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763367.236745:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2411 00000100:00080000:2.0:1787763367.236760:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763367.236768:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763367.236811:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274259072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763367.236844:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763367.236848:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763367.236850:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763367.236854:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2411 00000100:00080000:2.0:1787763367.236857:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763367.236877:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763367.236884:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259072:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787763367.236889:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763367.236903:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274259200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763367.236965:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763367.237013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763367/real 1787763367] req@ffff9ac4523bcf00 x1874603274259072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763367.237029:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274259072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763367.237039:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763367.237043:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763367.237046:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259072:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763367.237072:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259200:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763367.237089:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763367.237106:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763367/real 1787763367] req@ffff9ac4523bcb40 x1874603274259200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763367.237115:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274259200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763367.237122:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763367.237124:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763367.237127:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259200:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763367.812703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763368.260663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763368.836713:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763369.284682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763369.860693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763370.308612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763370.884720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763371.332701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763371.908745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763372.356697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763372.356758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763372.356768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763372.356773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763372.356785:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2417 00000100:00080000:3.0:1787763372.356799:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763372.356808:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763372.356856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274259328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763372.356928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763372.356936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763372.356940:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763372.356948:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2417 00000100:00080000:3.0:1787763372.356956:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763372.356965:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763372.356982:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763372.357008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274259456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763372.357020:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259328:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763372.357114:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763372.357168:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763372/real 1787763372] req@ffff9ac451c52940 x1874603274259328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763372.357189:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274259328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763372.357202:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763372.357207:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763372.357212:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259328:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763372.357241:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259456:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763372.357269:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763372.357284:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763372/real 1787763372] req@ffff9ac451c50b40 x1874603274259456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763372.357320:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274259456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763372.357331:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763372.357335:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763372.357339:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259456:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763372.932709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763373.380665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763373.956761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763374.404743:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763374.980707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763375.428652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763376.004824:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763376.452703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763377.028741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763377.476636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763377.476639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763377.476654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763377.476660:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763377.476671:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2422 00000100:00080000:3.0:1787763377.476686:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763377.476694:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763377.476744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274259584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763377.476776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763377.476780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763377.476782:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763377.476786:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2422 00000100:00080000:3.0:1787763377.476812:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763377.476816:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763377.476841:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763377.476854:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763377.476858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274259712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763377.476958:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763377.477011:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763377/real 1787763377] req@ffff9ac451c50780 x1874603274259584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763377.477039:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274259584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763377.477050:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763377.477054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763377.477058:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763377.477084:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259712:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763377.477103:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763377.477114:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763377/real 1787763377] req@ffff9ac451c503c0 x1874603274259712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763377.477125:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274259712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763377.477141:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763377.477145:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763377.477148:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259712:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763378.052658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763378.500632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763379.076669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763379.524622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763380.100699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763380.548600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763381.124668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763381.572680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763382.148665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763382.596591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763382.596603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787763382.596607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763382.596608:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763382.596621:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2427 00000100:00080000:3.0:1787763382.596635:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763382.596643:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763382.596689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274259840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763382.596714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763382.596717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763382.596719:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763382.596722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2427 00000100:00080000:3.0:1787763382.596724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763382.596727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763382.596737:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763382.596747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274259968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763382.596761:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259840:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763382.596843:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763382.596895:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763382/real 1787763382] req@ffff9ac451c52580 x1874603274259840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763382.596912:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274259840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763382.596922:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763382.596926:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763382.596929:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274259840:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763382.596956:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259968:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763382.596976:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763382.596988:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763382/real 1787763382] req@ffff9ac451c50f00 x1874603274259968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763382.597002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274259968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763382.597027:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763382.597032:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763382.597036:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274259968:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763383.172661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763383.620595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763384.196663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763384.644688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763385.220654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763385.668616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763386.244676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763386.692624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763387.268787:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763387.716648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763387.716660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763387.716675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763387.716682:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763387.716699:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2432 00000100:00080000:3.0:1787763387.716717:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763387.716730:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763387.716786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274260096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763387.716826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763387.716831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763387.716834:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763387.716839:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2432 00000100:00080000:3.0:1787763387.716845:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763387.716850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763387.716863:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763387.716881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274260224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763387.716906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260096:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763387.717018:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763387.717072:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763387/real 1787763387] req@ffff9ac451c50f00 x1874603274260096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763387.717089:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274260096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763387.717099:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763387.717103:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763387.717106:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260096:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763387.717136:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260224:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763387.717155:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763387.717165:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763387/real 1787763387] req@ffff9ac451c52580 x1874603274260224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763387.717174:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274260224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763387.717180:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763387.717203:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763387.717205:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260224:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763388.292784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763388.740685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763389.316661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763389.764626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763390.340662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763390.788605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763391.364727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763391.812632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763392.388780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763392.836606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763392.836608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763392.836618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763392.836658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763392.836668:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2437 00000100:00080000:3.0:1787763392.836682:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763392.836690:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763392.836732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274260352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763392.836758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763392.836761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763392.836763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763392.836766:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2437 00000100:00080000:3.0:1787763392.836769:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763392.836771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763392.836782:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763392.836791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274260480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763392.836850:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260352:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763392.836938:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763392.836982:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763392/real 1787763392] req@ffff9ac451c503c0 x1874603274260352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763392.836998:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274260352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763392.837008:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763392.837011:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763392.837014:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260352:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763392.837019:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260480:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763392.837035:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763392.837043:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763392/real 1787763392] req@ffff9ac451c50780 x1874603274260480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763392.837051:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274260480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763392.837057:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763392.837059:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763392.837061:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260480:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763393.412690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763393.860608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763394.436727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763394.884638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763395.460659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763395.908627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763396.484747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763396.932691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763397.508657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763397.956679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763397.956684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763397.956701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763397.956710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763397.956743:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2442 00000100:00080000:3.0:1787763397.956755:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763397.956764:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763397.956802:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274260608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763397.956848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763397.956853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763397.956855:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763397.956861:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2442 00000100:00080000:3.0:1787763397.956867:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763397.956873:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763397.956885:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763397.956895:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763397.956904:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274260736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763397.957013:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260736:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763398.532701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763398.980750:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763399.556757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763400.004732:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763400.580781:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763401.028815:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787763401.028968:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787763401.029045:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787763401.029067:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787763401.029142:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763401.029242:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763397/real 1787763401] req@ffff9ac451c52940 x1874603274260736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763401.029282:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274260736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763401.029300:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763401.029331:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763401.029339:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260736:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787763401.029353:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763397/real 1787763401] req@ffff9ac451c50b40 x1874603274260608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763401.029376:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274260608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763401.029388:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763401.029394:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763401.029399:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260608:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763401.604764:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763402.052708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763402.628667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763403.076635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763403.076645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763403.076649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763403.076659:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2447 00000100:00080000:3.0:1787763403.076671:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763403.076679:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763403.076689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787763403.076716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274260864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763403.076744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763403.076747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763403.076748:0:37:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763403.076752:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2447 00000100:00080000:3.0:1787763403.076768:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763403.076773:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763403.076782:0:37:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763403.076795:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763403.076797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274260992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763403.076867:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763403.076928:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763403/real 1787763403] req@ffff9ac451c52d00 x1874603274260864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763403.076951:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274260864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763403.076964:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763403.076969:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763403.076974:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274260864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763403.077003:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260992:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763403.077031:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763403.077094:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763403/real 1787763403] req@ffff9ac451c530c0 x1874603274260992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763403.077111:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274260992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763403.077123:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763403.077128:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763403.077133:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274260992:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763403.652757:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763404.100788:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763404.676756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763405.124640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763405.701330:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763406.148649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763406.727425:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763407.172648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763407.748717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763408.196649:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763408.196650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763408.196659:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763408.196663:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763408.196671:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2452 00000100:00080000:2.0:1787763408.196680:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763408.196686:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763408.196742:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274261120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763408.196777:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763408.196780:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763408.196781:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763408.196784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2452 00000100:00080000:2.0:1787763408.196788:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763408.196791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763408.196799:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763408.196809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274261248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763408.196836:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261120:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763408.196899:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763408.196932:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763408/real 1787763408] req@ffff9ac4523be1c0 x1874603274261120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763408.196956:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274261120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763408.196963:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763408.196966:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763408.196969:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763408.196974:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274261248:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763408.196990:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763408.196998:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763408/real 1787763408] req@ffff9ac4523bf0c0 x1874603274261248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763408.197006:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274261248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763408.197011:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763408.197013:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763408.197015:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274261248:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763408.772630:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763409.220635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763409.796766:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763410.244664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763410.820798:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763411.268636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763411.844746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763412.292608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763412.868740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763413.316661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763413.316662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763413.316676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763413.316682:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763413.316695:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2458 00000100:00080000:2.0:1787763413.316711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763413.316721:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763413.316768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274261376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763413.316817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763413.316822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763413.316824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763413.316830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2458 00000100:00080000:2.0:1787763413.316834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763413.316838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763413.316870:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763413.316888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274261504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763413.316920:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261376:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763413.317044:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763413.318877:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763413/real 1787763413] req@ffff9ac4523bf0c0 x1874603274261376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763413.318906:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274261376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763413.318940:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763413.318946:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763413.318952:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261376:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763413.318990:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274261504:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763413.319057:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763413.319075:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763413/real 1787763413] req@ffff9ac4523be1c0 x1874603274261504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763413.319107:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274261504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763413.319129:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763413.319133:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763413.319137:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274261504:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763413.892786:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763414.340687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763414.916750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763415.364623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763415.940752:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763416.388591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763416.964756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763417.412678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763417.988759:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763418.436666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763418.436710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763418.436717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763418.436722:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763418.436734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2463 00000100:00080000:3.0:1787763418.436747:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763418.436754:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763418.436833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274261632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763418.436904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763418.436908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763418.436910:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763418.436915:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2463 00000100:00080000:3.0:1787763418.436920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763418.436924:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763418.436937:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763418.436952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274261760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763418.437007:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261632:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763418.437096:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763418.437147:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763418/real 1787763418] req@ffff9ac451c530c0 x1874603274261632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763418.437165:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274261632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763418.437177:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763418.437181:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763418.437185:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261632:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763418.437219:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274261760:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763418.437245:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763418.437257:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763418/real 1787763418] req@ffff9ac451c52d00 x1874603274261760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763418.437268:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274261760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763418.437276:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763418.437279:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763418.437282:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274261760:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763419.012786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763419.460646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763420.036748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763420.484623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763421.060785:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763421.508600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763422.084769:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763422.532627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763423.108820:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763423.556599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763423.556686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763423.556701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763423.556706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763423.556719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2468 00000100:00080000:2.0:1787763423.556734:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763423.556744:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763423.556792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274261888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763423.556834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763423.556839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763423.556842:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763423.556848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2468 00000100:00080000:2.0:1787763423.556852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763423.556856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763423.556880:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763423.556900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274262016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763423.556923:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261888:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763423.557014:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763423.557060:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763423/real 1787763423] req@ffff9ac4523bcb40 x1874603274261888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763423.557076:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274261888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763423.557087:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763423.557091:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763423.557095:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274261888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763423.557117:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262016:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763423.557133:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763423.557141:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763423/real 1787763423] req@ffff9ac4523bcf00 x1874603274262016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763423.557194:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274262016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763423.557201:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763423.557203:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763423.557206:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262016:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763424.132800:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763424.580642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763425.156771:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763425.604610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763426.180724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763426.628620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763427.204827:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763427.652599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763428.228744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763428.676676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763428.676727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763428.676737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763428.676743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763428.676755:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2473 00000100:00080000:3.0:1787763428.676769:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763428.676778:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763428.676854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274262144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763428.676915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763428.676934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763428.676946:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763428.676956:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2473 00000100:00080000:3.0:1787763428.676966:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787763428.676976:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262144:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787763428.676978:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763428.677004:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763428.677029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274262272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763428.677081:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763428.677163:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763428/real 1787763428] req@ffff9ac451c50b40 x1874603274262144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763428.677187:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274262144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763428.677202:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763428.677207:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763428.677213:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262144:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763428.677241:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262272:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763428.677270:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763428.677285:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763428/real 1787763428] req@ffff9ac451c52940 x1874603274262272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763428.677311:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274262272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763428.677341:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763428.677346:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763428.677351:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262272:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763429.252762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763429.700658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763430.276773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763430.724595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763431.300806:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763431.748629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763432.324768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763432.772587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763433.348743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763433.796606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763433.796655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763433.796664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763433.796667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763433.796675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2478 00000100:00080000:2.0:1787763433.796704:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763433.796712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763433.796747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274262400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763433.796775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763433.796778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763433.796779:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763433.796782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2478 00000100:00080000:2.0:1787763433.796785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763433.796789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763433.796798:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763433.796810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274262528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763433.796826:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262400:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763433.796901:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763433.796946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763433/real 1787763433] req@ffff9ac4523be580 x1874603274262400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763433.796968:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274262400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763433.796982:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763433.796988:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763433.796992:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262400:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763433.797018:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262528:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763433.797037:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763433.797047:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763433/real 1787763433] req@ffff9ac4523bc780 x1874603274262528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763433.797056:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274262528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763433.797062:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763433.797064:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763433.797067:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262528:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763434.372773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763434.820654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763435.396786:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763435.844602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763436.420713:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763436.868682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763437.444735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763437.892667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763438.468822:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763438.916629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763438.916726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763438.916733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763438.916737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763438.916765:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2483 00000100:00080000:3.0:1787763438.916778:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763438.916785:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763438.916857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274262656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763438.916881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763438.916884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763438.916889:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763438.916893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2483 00000100:00080000:3.0:1787763438.916896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763438.916900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763438.916911:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763438.916924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274262784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763438.916946:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262656:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763438.917085:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763438.917127:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763438/real 1787763438] req@ffff9ac451c50780 x1874603274262656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763438.917142:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274262656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763438.917151:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763438.917154:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763438.917157:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262656:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763438.917163:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262784:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763438.917181:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763438.917190:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763438/real 1787763438] req@ffff9ac451c503c0 x1874603274262784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763438.917198:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274262784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763438.917204:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763438.917206:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763438.917208:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274262784:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763439.492809:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763439.940636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763440.516719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763440.964619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763441.540731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763441.988655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763442.564732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763443.012629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763443.588908:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763444.036631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763444.036679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763444.036691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763444.036697:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763444.036712:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2488 00000100:00080000:2.0:1787763444.036727:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763444.036736:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763444.036784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274262912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763444.036823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763444.036828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763444.036850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763444.036856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2488 00000100:00080000:2.0:1787763444.036862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763444.036872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763444.036889:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763444.036914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262912:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763444.036927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274263040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763444.037026:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763444.037080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763444/real 1787763444] req@ffff9ac4523bd680 x1874603274262912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763444.037106:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274262912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763444.037116:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763444.037120:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763444.037123:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274262912:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763444.037150:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263040:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763444.037170:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763444.037180:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763444/real 1787763444] req@ffff9ac4523bc000 x1874603274263040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763444.037194:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274263040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763444.037204:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763444.037208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763444.037211:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263040:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763444.612765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763445.060637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763445.636781:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763446.084664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763446.660754:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763447.108606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763447.684713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763448.132681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763448.708787:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763449.156663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763449.156696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763449.156711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763449.156718:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763449.156734:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2493 00000100:00080000:2.0:1787763449.156748:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763449.156757:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763449.156801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274263168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763449.156838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763449.156843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763449.156846:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763449.156851:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2493 00000100:00080000:2.0:1787763449.156855:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763449.156860:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763449.156871:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763449.156886:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274263296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763449.156969:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263168:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763449.157115:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763449.157174:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763449/real 1787763449] req@ffff9ac4523be940 x1874603274263168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763449.157208:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274263168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763449.157256:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763449.157266:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763449.157274:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263168:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763449.157288:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263296:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763449.157326:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763449.157345:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763449/real 1787763449] req@ffff9ac4523bc3c0 x1874603274263296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763449.157361:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274263296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763449.157372:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763449.157377:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763449.157381:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263296:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763449.732771:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763450.180645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763450.756764:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763451.204706:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763451.780782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763452.228600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763452.804780:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763453.252648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763453.828710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763454.276621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763454.276654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763454.276666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763454.276672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763454.276684:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2499 00000100:00080000:2.0:1787763454.276699:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763454.276720:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763454.276769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274263424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763454.276807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763454.276813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763454.276815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763454.276821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2499 00000100:00080000:2.0:1787763454.276826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763454.276834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763454.276849:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763454.276864:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763454.276869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274263552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763454.276980:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763454.277029:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763454/real 1787763454] req@ffff9ac4523bf840 x1874603274263424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763454.277045:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274263424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763454.277055:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763454.277059:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763454.277062:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763454.277110:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263552:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763454.277133:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763454.277148:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763454/real 1787763454] req@ffff9ac4523bda40 x1874603274263552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763454.277163:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274263552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763454.277174:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763454.277178:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763454.277181:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263552:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763454.852772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763455.300628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763455.876811:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763456.324650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763456.900788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763457.348606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763457.924779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763458.372628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763458.948789:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763459.396667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763459.396744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763459.396754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763459.396759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763459.396771:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2504 00000100:00080000:3.0:1787763459.396785:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763459.396818:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763459.396859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274263680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763459.396897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763459.396902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763459.396934:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763459.396941:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2504 00000100:00080000:3.0:1787763459.396949:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763459.396956:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763459.396970:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763459.396982:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263680:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763459.396994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274263808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763459.397082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763459.397133:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763459/real 1787763459] req@ffff9ac451c52580 x1874603274263680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763459.397176:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274263680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763459.397196:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763459.397205:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763459.397214:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263680:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763459.397254:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263808:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763459.397288:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763459.397306:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763459/real 1787763459] req@ffff9ac451c50f00 x1874603274263808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763459.397336:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274263808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763459.397362:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763459.397366:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763459.397371:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274263808:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763459.972735:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763460.420635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763460.996678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763461.444609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763462.020784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763462.468603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763463.044709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763463.492668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763464.068777:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763464.516603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763464.516622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763464.516628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763464.516633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763464.516643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2509 00000100:00080000:2.0:1787763464.516655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763464.516662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763464.516723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274263936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763464.516748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763464.516751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763464.516752:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763464.516755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2509 00000100:00080000:2.0:1787763464.516758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763464.516761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763464.516770:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763464.516780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274264064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763464.516788:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263936:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763464.516910:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264064:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763465.092721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763465.540699:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763466.116669:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763466.564735:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763467.140671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763467.588842:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787763467.588985:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787763467.589031:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787763467.589066:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787763467.589112:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763467.589590:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763464/real 1787763467] req@ffff9ac4523bf840 x1874603274264064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763467.589621:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274264064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763467.589637:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763467.589644:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763467.589649:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264064:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787763467.589660:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763464/real 1787763467] req@ffff9ac4523bda40 x1874603274263936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763467.589675:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274263936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763467.589685:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763467.589689:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763467.589713:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274263936:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763468.164798:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763468.612742:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763469.188745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763469.636624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763469.636634:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763469.636638:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763469.636649:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2514 00000100:00080000:0.0:1787763469.636672:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763469.636681:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763469.636710:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787763469.636723:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274264192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763469.636756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763469.636759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763469.636760:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763469.636764:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2514 00000100:00080000:0.0:1787763469.636766:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763469.636769:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763469.636779:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763469.636789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274264320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763469.636805:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264192:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763469.636922:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763469.636983:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763469/real 1787763469] req@ffff9ac451f870c0 x1874603274264192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763469.637005:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274264192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763469.637019:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763469.637026:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763469.637050:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264192:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763469.637081:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264320:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763469.637111:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763469.637128:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763469/real 1787763469] req@ffff9ac451f84000 x1874603274264320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763469.637141:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274264320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763469.637150:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763469.637153:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763469.637156:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264320:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763470.212727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763470.660870:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763471.236763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763471.684612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763472.260857:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763472.708653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763473.284756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763473.732598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763474.308703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763474.756646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763474.756653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763474.756660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763474.756667:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763474.756698:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2519 00000100:00080000:0.0:1787763474.756712:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763474.756721:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763474.756774:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274264448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763474.756830:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763474.756834:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763474.756836:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763474.756842:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2519 00000100:00080000:0.0:1787763474.756846:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763474.756850:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763474.756874:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763474.756889:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274264576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763474.756916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264448:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763474.756999:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763474.757046:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763474/real 1787763474] req@ffff9ac451f84b40 x1874603274264448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763474.757064:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274264448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763474.757074:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763474.757078:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763474.757082:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264448:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763474.757090:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264576:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763474.757110:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763474.757130:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763474/real 1787763474] req@ffff9ac451f87c00 x1874603274264576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763474.757140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274264576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763474.757148:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763474.757150:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763474.757154:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264576:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763475.332670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763475.780619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763476.356750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763476.804626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763477.380723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763477.828637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763478.404727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763478.852667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763479.428745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763479.876641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763479.876659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763479.876665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763479.876670:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763479.876681:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2524 00000100:00080000:3.0:1787763479.876692:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763479.876700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763479.876744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274264704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763479.876818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763479.876822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763479.876823:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763479.876827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2524 00000100:00080000:3.0:1787763479.876830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763479.876834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763479.876845:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763479.876858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274264832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763479.876863:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264704:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763479.876942:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763479.876987:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763479/real 1787763479] req@ffff9ac451c50f00 x1874603274264704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763479.877002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274264704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763479.877012:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763479.877026:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763479.877030:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264704:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763479.877055:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264832:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763479.877072:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763479.877080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763479/real 1787763479] req@ffff9ac451c52580 x1874603274264832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763479.877089:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274264832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763479.877106:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763479.877110:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763479.877114:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274264832:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763480.452802:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763480.900670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763481.476695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763481.924612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763482.500770:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763482.948622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763483.524760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763483.972661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763484.548671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763484.996617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763484.996635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763484.996644:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763484.996651:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763484.996663:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2529 00000100:00080000:0.0:1787763484.996675:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763484.996683:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763484.996767:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274264960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763484.996795:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763484.996798:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763484.996799:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763484.996803:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2529 00000100:00080000:0.0:1787763484.996807:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763484.996811:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763484.996822:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763484.996829:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264960:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763484.996835:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274265088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763484.996911:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763484.996961:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763484/real 1787763484] req@ffff9ac451f87c00 x1874603274264960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763484.996985:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274264960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763484.996995:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763484.996999:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763484.997002:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274264960:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763484.997028:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265088:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763484.997045:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763484.997054:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763484/real 1787763484] req@ffff9ac451f84b40 x1874603274265088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763484.997062:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274265088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763484.997075:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763484.997078:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763484.997081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265088:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763485.572711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763486.020615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763486.596762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763487.044691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763487.620843:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763488.068675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763488.644762:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763489.092744:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763489.668795:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763490.116637:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763490.116650:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763490.116657:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787763490.116660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763490.116671:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2534 00000100:00080000:2.0:1787763490.116688:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763490.116710:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763490.116760:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274265216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763490.116789:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763490.116793:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763490.116794:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763490.116798:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2534 00000100:00080000:2.0:1787763490.116802:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763490.116806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763490.116818:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763490.116830:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265216:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763490.116831:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274265344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763490.116925:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763490.116983:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763490/real 1787763490] req@ffff9ac4523bc3c0 x1874603274265216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763490.117003:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274265216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763490.117013:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763490.117016:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763490.117019:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265216:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763490.117044:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265344:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763490.117072:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763490.117082:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763490/real 1787763490] req@ffff9ac4523be940 x1874603274265344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763490.117090:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274265344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763490.117097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763490.117099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763490.117102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265344:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763490.692709:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763491.140591:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763491.716665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763492.164616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763492.740805:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763493.188622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763493.764748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763494.212624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763494.788773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763495.236715:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763495.236737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763495.236746:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763495.236751:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763495.236761:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2539 00000100:00080000:0.0:1787763495.236774:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763495.236782:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763495.236832:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274265472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763495.236868:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763495.236872:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763495.236874:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763495.236879:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2539 00000100:00080000:0.0:1787763495.236883:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763495.236905:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763495.236917:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763495.236945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274265600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763495.236965:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265472:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763495.237083:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763495.237131:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763495/real 1787763495] req@ffff9ac451f84000 x1874603274265472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763495.237148:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274265472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763495.237159:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763495.237164:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763495.237169:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265472:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763495.237202:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265600:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763495.237231:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763495.237244:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763495/real 1787763495] req@ffff9ac451f870c0 x1874603274265600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763495.237254:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274265600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763495.237261:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763495.237264:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763495.237267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265600:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763495.812715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763496.260670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763496.836707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763497.284602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763497.860761:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763498.308607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763498.884707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763499.332600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763499.908688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763500.356641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763500.356664:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763500.356676:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763500.356681:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763500.356692:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2545 00000100:00080000:0.0:1787763500.356704:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763500.356711:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763500.356748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274265728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763500.356780:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763500.356784:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763500.356786:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763500.356791:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2545 00000100:00080000:0.0:1787763500.356795:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763500.356801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763500.356812:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763500.356828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274265856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763500.356841:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265728:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763500.356919:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763500.356979:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763500/real 1787763500] req@ffff9ac451f861c0 x1874603274265728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763500.356994:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274265728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763500.357004:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763500.357008:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763500.357011:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265728:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763500.357033:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265856:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763500.357050:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763500.357058:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763500/real 1787763500] req@ffff9ac451f84780 x1874603274265856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763500.357067:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274265856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763500.357073:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763500.357075:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763500.357077:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274265856:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763500.932708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763501.380616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763501.956743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763502.404622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763502.980707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763503.428637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763504.004762:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763504.452653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763505.028788:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763505.476609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763505.476637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763505.476643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763505.476646:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763505.476654:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2550 00000100:00080000:2.0:1787763505.476663:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763505.476669:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763505.476702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274265984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763505.476741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763505.476744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763505.476746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763505.476749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2550 00000100:00080000:2.0:1787763505.476752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763505.476755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763505.476763:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763505.476774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274266112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763505.476779:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265984:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763505.476841:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763505.476875:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763505/real 1787763505] req@ffff9ac4523bc000 x1874603274265984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763505.476888:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274265984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763505.476895:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763505.476898:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763505.476901:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274265984:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763505.476919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266112:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763505.476934:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763505.476942:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763505/real 1787763505] req@ffff9ac4523bd680 x1874603274266112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763505.476950:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274266112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763505.476956:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763505.476958:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763505.476960:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266112:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763506.052720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763506.500610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763507.076719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763507.524654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763508.100693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763508.548686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763509.124660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763509.572611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763510.148668:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763510.596606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763510.596688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763510.596747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763510.596863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763510.596876:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2555 00000100:00080000:3.0:1787763510.596904:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763510.597154:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763510.597232:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274266240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763510.597254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763510.597258:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763510.597259:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763510.597283:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2555 00000100:00100000:0.0:1787763510.597284:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274266240:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787763510.597287:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763510.597294:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763510.597304:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763510.597318:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274266368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763510.597363:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763510.597401:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763510/real 1787763510] req@ffff9ac451c503c0 x1874603274266240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763510.597428:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274266240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763510.597437:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763510.597440:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763510.597443:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274266240:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763510.597464:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266368:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763510.597482:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763510.597491:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763510/real 1787763510] req@ffff9ac451c50780 x1874603274266368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763510.597519:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274266368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763510.597526:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763510.597571:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763510.597576:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266368:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763511.172712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763511.620631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763512.196709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763512.644626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763513.220707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763513.668603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763514.244687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763514.692609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763515.268675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763515.716605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763515.716610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763515.716613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763515.716618:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763515.716627:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2560 00000100:00080000:3.0:1787763515.716636:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763515.716643:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763515.716672:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274266496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763515.716693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763515.716696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763515.716697:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763515.716700:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2560 00000100:00080000:3.0:1787763515.716702:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763515.716705:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763515.716713:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763515.716734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274266624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763515.716752:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274266496:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763515.716830:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763515.716876:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763515/real 1787763515] req@ffff9ac451c52940 x1874603274266496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763515.716892:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274266496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763515.716917:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763515.716921:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763515.716925:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274266496:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763515.716950:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266624:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763515.716969:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763515.716978:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763515/real 1787763515] req@ffff9ac451c50b40 x1874603274266624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763515.716986:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274266624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763515.716992:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763515.716995:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763515.716997:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266624:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763516.292682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763516.740604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763517.316659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763517.764617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763518.340700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763518.788610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763519.364684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763519.812655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763520.388652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763520.836609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763520.836635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763520.836644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763520.836649:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763520.836660:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2565 00000100:00080000:2.0:1787763520.836672:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763520.836679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763520.836723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274266752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763520.836751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763520.836754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763520.836756:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763520.836760:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2565 00000100:00080000:2.0:1787763520.836762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763520.836765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763520.836795:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763520.836796:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274266752:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763520.836814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274266880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763520.836867:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763520.836901:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763520/real 1787763520] req@ffff9ac4523bc780 x1874603274266752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763520.836914:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274266752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763520.836922:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763520.836926:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763520.836929:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274266752:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763520.836962:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266880:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763520.836980:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763520.836995:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763520/real 1787763520] req@ffff9ac4523be580 x1874603274266880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763520.837003:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274266880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763520.837009:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763520.837011:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763520.837014:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274266880:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763521.412705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763521.860615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763522.436675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763522.884602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763523.460676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763523.908603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763524.484688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763524.932622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763525.508676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763525.956617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763525.956618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763525.956628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763525.956633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763525.956643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2570 00000100:00080000:2.0:1787763525.956654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763525.956662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763525.956705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274267008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763525.956731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763525.956734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763525.956735:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763525.956738:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2570 00000100:00080000:2.0:1787763525.956741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763525.956743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763525.956753:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763525.956762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274267136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763525.956807:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267008:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763525.956878:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763525.956921:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763525/real 1787763525] req@ffff9ac4523bcf00 x1874603274267008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763525.956935:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274267008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763525.956944:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763525.956947:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763525.956950:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267008:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763525.956955:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267136:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763525.956970:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763525.956977:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763525/real 1787763525] req@ffff9ac4523bcb40 x1874603274267136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763525.957007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274267136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763525.957013:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763525.957016:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763525.957018:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267136:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763526.532706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763526.980680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763527.556705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763528.004649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763528.580743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763529.028593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763529.604707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763530.052674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763530.628702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763531.076612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763531.076617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763531.076625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763531.076630:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763531.076646:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2575 00000100:00080000:3.0:1787763531.076658:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763531.076666:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763531.076707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274267264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763531.076730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763531.076733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763531.076735:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763531.076738:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2575 00000100:00080000:3.0:1787763531.076740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763531.076743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763531.076752:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763531.076761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274267392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763531.076803:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267264:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763531.076912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267392:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763531.652653:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763532.100605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763532.676688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763533.124649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763533.700707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763534.148644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787763534.148863:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787763534.148922:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787763534.148947:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787763534.149021:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763534.149068:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763531/real 1787763534] req@ffff9ac451c530c0 x1874603274267392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763534.149099:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274267392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763534.149114:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763534.149122:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763534.149127:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267392:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787763534.149137:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763531/real 1787763534] req@ffff9ac451c52d00 x1874603274267264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763534.149149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274267264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763534.149156:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763534.149158:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763534.149168:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267264:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787763534.724751:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763535.172605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763535.748722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763536.196602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763536.196690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763536.196698:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763536.196704:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763536.196717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2580 00000100:00080000:2.0:1787763536.196731:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763536.196741:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763536.196791:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274267520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763536.196824:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763536.196828:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763536.196830:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763536.196835:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2580 00000100:00080000:2.0:1787763536.196839:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763536.196844:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763536.196857:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763536.196874:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274267648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763536.196953:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267520:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763536.197061:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763536.197145:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763536/real 1787763536] req@ffff9ac4523be1c0 x1874603274267520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763536.197181:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274267520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763536.197203:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763536.197212:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763536.197221:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267520:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763536.197237:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267648:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763536.197285:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763536.197307:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763536/real 1787763536] req@ffff9ac4523bf0c0 x1874603274267648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763536.197335:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274267648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763536.197352:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763536.197359:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763536.197366:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267648:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763536.772733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763537.220630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763537.796680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763538.244582:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763538.821076:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763539.268652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763539.844836:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763540.292615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763540.868793:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763541.316670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763541.316682:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763541.316695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763541.316701:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763541.316717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2586 00000100:00080000:2.0:1787763541.316734:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763541.316745:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763541.316819:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274267776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763541.316854:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763541.316859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763541.316862:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763541.316868:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2586 00000100:00080000:2.0:1787763541.316874:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763541.316882:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763541.316897:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763541.316913:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274267904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763541.316926:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267776:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763541.317023:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763541.317083:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763541/real 1787763541] req@ffff9ac4523bf0c0 x1874603274267776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763541.317100:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274267776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763541.317110:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763541.317114:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763541.317118:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274267776:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763541.317147:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267904:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763541.317167:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763541.317176:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763541/real 1787763541] req@ffff9ac4523be1c0 x1874603274267904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763541.317185:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274267904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763541.317192:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763541.317194:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763541.317196:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274267904:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763541.892686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763542.340654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763542.916817:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763543.364669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763543.941621:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763544.388608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763544.964696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763545.412648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763545.988704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763546.436655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763546.436692:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763546.436703:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763546.436709:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763546.436722:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2591 00000100:00080000:0.0:1787763546.436757:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763546.436768:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763546.436814:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274268032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763546.436862:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763546.436871:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763546.436876:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763546.436886:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2591 00000100:00080000:0.0:1787763546.436895:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763546.436907:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763546.436919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268032:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763546.436925:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763546.436961:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274268160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763546.437013:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763546.437065:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763546/real 1787763546] req@ffff9ac451f85680 x1874603274268032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763546.437088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274268032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763546.437103:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763546.437108:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763546.437113:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268032:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763546.437143:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268160:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763546.437171:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763546.437204:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763546/real 1787763546] req@ffff9ac451f843c0 x1874603274268160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763546.437220:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274268160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763546.437232:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763546.437237:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763546.437241:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268160:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763547.012739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763547.460638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763548.036637:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763548.484647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763549.060766:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763549.508632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763550.084707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763550.532659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763551.108721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763551.556652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763551.556657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763551.556660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763551.556669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763551.556679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2596 00000100:00080000:3.0:1787763551.556688:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763551.556695:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763551.556727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274268288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763551.556750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763551.556752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763551.556754:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763551.556757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2596 00000100:00080000:3.0:1787763551.556759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763551.556761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763551.556768:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763551.556777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274268416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763551.556817:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268288:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763551.556877:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763551.556910:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763551/real 1787763551] req@ffff9ac451c52d00 x1874603274268288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763551.556923:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274268288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763551.556931:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763551.556934:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763551.556937:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268288:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763551.556942:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268416:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763551.556956:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763551.556964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763551/real 1787763551] req@ffff9ac451c530c0 x1874603274268416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763551.556971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274268416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763551.556993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763551.556996:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763551.556998:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268416:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763552.132696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763552.580625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763553.156674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763553.604623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763554.180654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763554.628633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763555.204708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763555.652595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763556.228784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763556.676659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787763556.676664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763556.676670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763556.676711:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763556.676723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2601 00000100:00080000:3.0:1787763556.676736:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763556.676744:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763556.676787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274268544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763556.676814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763556.676828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763556.676829:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763556.676833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2601 00000100:00080000:3.0:1787763556.676835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763556.676837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763556.676847:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763556.676856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274268672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763556.676882:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268544:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763556.676954:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763556.676995:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763556/real 1787763556] req@ffff9ac451c50b40 x1874603274268544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763556.677009:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274268544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763556.677017:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763556.677021:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763556.677024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268544:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763556.677029:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268672:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763556.677044:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763556.677053:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763556/real 1787763556] req@ffff9ac451c52940 x1874603274268672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763556.677061:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274268672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763556.677067:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763556.677068:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763556.677070:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268672:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763557.252678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763557.700609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763558.276688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763558.724657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763559.300706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763559.748651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763560.324680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763560.772631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763561.348772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763561.796606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763561.796615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763561.796621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763561.796624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763561.796632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2606 00000100:00080000:0.0:1787763561.796642:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763561.796648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763561.796694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274268800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763561.796716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763561.796718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763561.796719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763561.796723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2606 00000100:00080000:0.0:1787763561.796725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763561.796728:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763561.796735:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763561.796743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274268928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763561.796826:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268800:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763561.796918:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763561.796977:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763561/real 1787763561] req@ffff9ac451f843c0 x1874603274268800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763561.796993:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274268800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763561.797002:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763561.797006:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763561.797009:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274268800:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763561.797015:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268928:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763561.797034:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763561.797042:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763561/real 1787763561] req@ffff9ac451f85680 x1874603274268928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763561.797050:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274268928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763561.797056:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763561.797058:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763561.797060:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274268928:192.168.203.123@tcp:38:kworker.0 00010000:00010000:0.0:1787763562.372758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763562.820607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763563.396682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763563.844667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763564.420765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763564.868584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763565.444655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763565.892662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787763566.468725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763566.916641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763566.916692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763566.916701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763566.916706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763566.916744:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2611 00000100:00080000:2.0:1787763566.916756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763566.916763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763566.916803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274269056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763566.916833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763566.916836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763566.916838:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763566.916841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2611 00000100:00080000:2.0:1787763566.916844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763566.916847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763566.916856:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763566.916867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763566.916868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274269184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763566.916941:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763566.916983:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763566/real 1787763566] req@ffff9ac4523bcb40 x1874603274269056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763566.916996:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274269056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763566.917005:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763566.917008:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763566.917011:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763566.917041:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269184:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763566.917059:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763566.917068:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763566/real 1787763566] req@ffff9ac4523bcf00 x1874603274269184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763566.917076:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274269184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763566.917082:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763566.917084:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763566.917087:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269184:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763567.492674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763567.940602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763568.516724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763568.964611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763569.540682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763569.988602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763570.564702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763571.012629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763571.588697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763572.036609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763572.036625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763572.036632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763572.036636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763572.036647:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2616 00000100:00080000:0.0:1787763572.036660:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763572.036668:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763572.036711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274269312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763572.036740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763572.036743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763572.036745:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763572.036754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2616 00000100:00080000:0.0:1787763572.036757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763572.036776:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269312:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787763572.036783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763572.036794:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763572.036808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274269440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763572.036853:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763572.036894:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763572/real 1787763572] req@ffff9ac451f84780 x1874603274269312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763572.036909:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274269312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763572.036919:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763572.036922:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763572.036925:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269312:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763572.036954:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269440:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763572.036972:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763572.036982:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763572/real 1787763572] req@ffff9ac451f861c0 x1874603274269440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763572.036996:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274269440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763572.037004:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763572.037008:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763572.037012:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269440:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763572.612706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763573.060612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763573.636660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763574.084596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763574.660782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763575.108599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763575.684703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763576.132616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763576.708705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763577.156593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763577.156702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763577.156710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763577.156715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763577.156726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2621 00000100:00080000:2.0:1787763577.156738:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763577.156746:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763577.156790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274269568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763577.156823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763577.156827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763577.156828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763577.156832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2621 00000100:00080000:2.0:1787763577.156836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763577.156862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763577.156863:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269568:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787763577.156878:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763577.156893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274269696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763577.156932:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763577.156965:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763577/real 1787763577] req@ffff9ac4523be580 x1874603274269568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763577.156978:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274269568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763577.156986:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763577.156989:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763577.156992:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269568:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763577.157012:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269696:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763577.157033:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763577.157047:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763577/real 1787763577] req@ffff9ac4523bc780 x1874603274269696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763577.157060:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274269696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763577.157070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763577.157074:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763577.157078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269696:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763577.732739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763578.180595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763578.756716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763579.204603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763579.780750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763580.228632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763580.804706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763581.252613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763581.828714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763582.276627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763582.276688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763582.276694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763582.276713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763582.276723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2627 00000100:00080000:1.0:1787763582.276734:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763582.276742:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763582.276773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274269824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763582.276803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763582.276807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763582.276809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763582.276814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2627 00000100:00080000:1.0:1787763582.276819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763582.276825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763582.276834:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763582.276856:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269824:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763582.276857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274269952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763582.276952:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763582.276995:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763582/real 1787763582] req@ffff9ac447840b40 x1874603274269824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763582.277011:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274269824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763582.277021:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763582.277024:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763582.277028:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274269824:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763582.277050:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269952:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763582.277084:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763582.277094:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763582/real 1787763582] req@ffff9ac447842940 x1874603274269952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763582.277103:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274269952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763582.277110:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763582.277113:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763582.277116:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274269952:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763582.852746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763583.300592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763583.876724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763584.324613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763584.900764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763585.348596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763585.924697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763586.372605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763586.948712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763587.396646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763587.396660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763587.396666:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763587.396679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2632 00000100:00080000:1.0:1787763587.396692:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763587.396702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763587.396714:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787763587.396774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274270080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763587.396806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763587.396811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763587.396813:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763587.396819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2632 00000100:00080000:1.0:1787763587.396824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763587.396829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763587.396842:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763587.396852:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270080:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763587.396862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274270208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763587.396924:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763587.396961:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763587/real 1787763587] req@ffff9ac447843c00 x1874603274270080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763587.396974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274270080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763587.396983:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763587.396986:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763587.396989:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270080:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763587.397007:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270208:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763587.397023:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763587.397058:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763587/real 1787763587] req@ffff9ac447842940 x1874603274270208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763587.397067:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274270208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763587.397073:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763587.397076:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763587.397078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270208:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763587.972717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763588.420644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763588.996699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763589.444652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763590.020676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763590.468641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763591.044711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763591.492682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763592.068723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763592.516621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763592.516629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763592.516633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763592.516641:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2637 00000100:00100000:2.0:1787763592.516645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763592.516651:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763592.516659:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763592.516690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274270336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763592.516709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763592.516712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763592.516713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763592.516716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2637 00000100:00080000:1.0:1787763592.516732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763592.516735:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270336:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763592.516739:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763592.516750:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763592.516763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274270464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763592.516797:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763592.516831:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763592/real 1787763592] req@ffff9ac447840b40 x1874603274270336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763592.516844:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274270336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763592.516851:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763592.516854:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763592.516857:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763592.516874:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270464:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763592.516890:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763592.516898:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763592/real 1787763592] req@ffff9ac447842d00 x1874603274270464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763592.516906:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274270464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763592.516911:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763592.516914:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763592.516916:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270464:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763593.092645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763593.540659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763594.116745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763594.564647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763595.140750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763595.588632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763596.164674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763596.612597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763597.188758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763597.636596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763597.636596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763597.636604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763597.636608:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763597.636615:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2642 00000100:00080000:1.0:1787763597.636625:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763597.636630:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763597.636661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274270592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763597.636681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763597.636684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763597.636686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763597.636688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2642 00000100:00080000:1.0:1787763597.636691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763597.636694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763597.636701:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763597.636714:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270592:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763597.636715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274270720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763597.636808:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270720:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763598.212695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763598.660650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763599.236716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763599.684636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763600.260702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763600.708641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763600.708833:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763600.708878:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763600.708912:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763600.708970:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787763600.709013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763597/real 1787763600] req@ffff9ac4478421c0 x1874603274270720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763600.709044:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274270720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763600.709055:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763600.709064:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763600.709069:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270720:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787763600.709076:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763597/real 1787763600] req@ffff9ac4478430c0 x1874603274270592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763600.709089:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274270592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763600.709097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763600.709101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763600.709104:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270592:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763601.284731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763601.732708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763602.308700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763602.756614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763602.756621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763602.756625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763602.756634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2647 00000100:00080000:0.0:1787763602.756644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763602.756650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763602.756669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787763602.756681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274270848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763602.756704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763602.756707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763602.756709:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763602.756712:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2647 00000100:00080000:0.0:1787763602.756714:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763602.756716:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763602.756724:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763602.756733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274270976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763602.756751:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270848:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763602.756808:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763602.756856:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763602/real 1787763602] req@ffff9ac451f870c0 x1874603274270848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763602.756869:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274270848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763602.756877:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763602.756882:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763602.756887:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274270848:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763602.756912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270976:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763602.756952:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763602.756967:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763602/real 1787763602] req@ffff9ac451f84000 x1874603274270976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763602.756978:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274270976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763602.756987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763602.756990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763602.756994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274270976:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763603.332795:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763603.780709:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763604.356652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763604.804646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763605.380827:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763605.828656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763606.404709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763606.852611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763607.428660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763607.876622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763607.876630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763607.876634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763607.876642:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2652 00000100:00080000:0.0:1787763607.876654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763607.876660:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763607.876686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787763607.876696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274271104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763607.876742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763607.876747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763607.876749:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763607.876756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2652 00000100:00080000:0.0:1787763607.876760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763607.876766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763607.876777:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763607.876792:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763607.876795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274271232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763607.876850:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763607.876896:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763607/real 1787763607] req@ffff9ac451f84b40 x1874603274271104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763607.876912:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274271104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763607.876926:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763607.876929:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763607.876932:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763607.876950:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274271232:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763607.876968:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763607.876984:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763607/real 1787763607] req@ffff9ac451f87c00 x1874603274271232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763607.876993:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274271232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763607.876999:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763607.877001:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763607.877004:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274271232:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763608.452685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763608.900652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763609.476675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763609.924609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763610.500682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763610.948625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763611.524727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787763611.972679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763612.548746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763612.996638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763612.996650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763612.996667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763612.996679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2657 00000100:00080000:1.0:1787763612.996693:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787763612.996697:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763612.996702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763612.996750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274271360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763612.996786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763612.996789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763612.996791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763612.996794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2657 00000100:00080000:1.0:1787763612.996797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763612.996799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763612.996810:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763612.996820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274271488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763612.996861:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271360:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763612.996935:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763612.996979:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763612/real 1787763612] req@ffff9ac4478403c0 x1874603274271360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763612.996995:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274271360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763612.997025:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763612.997036:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763612.997040:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271360:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763612.997062:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274271488:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763612.997082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763612.997092:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763612/real 1787763612] req@ffff9ac447841680 x1874603274271488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763612.997110:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274271488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763612.997116:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763612.997119:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763612.997121:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274271488:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763613.572700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763614.020601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763614.596723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763615.044671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763615.620756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763616.068618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763616.644692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763617.092633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763617.668673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763618.116634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763618.116643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763618.116651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763618.116679:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763618.116690:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2662 00000100:00080000:1.0:1787763618.116714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763618.116721:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763618.116761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274271616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763618.116799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763618.116802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763618.116804:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763618.116807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2662 00000100:00080000:1.0:1787763618.116809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763618.116812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763618.116852:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763618.116865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274271744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763618.116948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271616:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763618.117086:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763618.117161:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763618/real 1787763618] req@ffff9ac447840780 x1874603274271616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763618.117184:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274271616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763618.117212:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763618.117217:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763618.117222:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271616:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763618.117231:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274271744:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763618.117350:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763618.117366:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763618/real 1787763618] req@ffff9ac447840f00 x1874603274271744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763618.117631:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274271744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763618.117646:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763618.117650:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763618.117655:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274271744:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763618.692743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763619.140587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763619.716731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763620.164616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763620.740765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763621.188658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763621.764774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763622.212663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763622.788721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763623.236614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763623.236687:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763623.236697:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763623.236700:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763623.236709:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2667 00000100:00080000:2.0:1787763623.236736:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763623.236747:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763623.236807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274271872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763623.236854:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763623.236857:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763623.236858:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763623.236862:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2667 00000100:00080000:2.0:1787763623.236865:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763623.236868:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763623.236876:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763623.236889:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274272000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763623.236910:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271872:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763623.236988:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763623.237045:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763623/real 1787763623] req@ffff9ac4523bd680 x1874603274271872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763623.237061:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274271872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763623.237099:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763623.237102:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763623.237106:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274271872:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763623.237133:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272000:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763623.237153:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763623.237162:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763623/real 1787763623] req@ffff9ac4523bc000 x1874603274272000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763623.237171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274272000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763623.237177:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763623.237180:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763623.237182:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272000:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763623.812806:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763624.260595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763624.836747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763625.284637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763625.860808:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763626.308618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763626.884694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763627.332631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763627.908685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763628.356614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763628.356617:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763628.356625:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763628.356628:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763628.356637:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2673 00000100:00080000:2.0:1787763628.356647:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763628.356653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763628.356684:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274272128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763628.356716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763628.356719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763628.356720:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763628.356723:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2673 00000100:00080000:2.0:1787763628.356726:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763628.356729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763628.356737:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763628.356760:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274272256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763628.356769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272128:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763628.356826:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763628.356859:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763628/real 1787763628] req@ffff9ac4523be940 x1874603274272128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763628.356873:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274272128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763628.356887:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763628.356890:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763628.356893:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272128:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763628.356914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272256:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763628.356939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763628.356954:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763628/real 1787763628] req@ffff9ac4523bc3c0 x1874603274272256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763628.356969:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274272256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763628.356975:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763628.356978:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763628.356980:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272256:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763628.932788:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763629.380628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763629.956700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763630.404588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763630.980781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763631.428622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763632.004819:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763632.452608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763633.028817:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763633.476601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763633.476665:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763633.476671:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763633.476675:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763633.476685:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2678 00000100:00080000:3.0:1787763633.476697:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763633.476704:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763633.476775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274272384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763633.476803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763633.476807:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763633.476808:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763633.476811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2678 00000100:00080000:3.0:1787763633.476815:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763633.476819:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763633.476828:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763633.476839:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274272512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763633.476862:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272384:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763633.476929:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763633.476965:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763633/real 1787763633] req@ffff9ac451c50780 x1874603274272384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763633.476980:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274272384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763633.476988:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763633.476991:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763633.476994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272384:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763633.477013:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272512:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763633.477029:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763633.477036:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763633/real 1787763633] req@ffff9ac451c503c0 x1874603274272512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763633.477059:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274272512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763633.477065:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763633.477067:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763633.477069:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272512:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763634.052749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763634.500657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763635.076783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763635.524629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763636.100789:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763636.548649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763637.124746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763637.572609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763638.148737:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763638.596696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763638.596699:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763638.596724:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763638.596728:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763638.596737:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2683 00000100:00080000:3.0:1787763638.596747:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763638.596755:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763638.596797:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274272640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763638.596820:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763638.596838:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763638.596840:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763638.596844:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2683 00000100:00080000:3.0:1787763638.596847:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763638.596851:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763638.596861:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763638.596872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274272768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763638.596891:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272640:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763638.596967:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763638.597014:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763638/real 1787763638] req@ffff9ac451c52580 x1874603274272640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763638.597036:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274272640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763638.597049:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763638.597054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763638.597058:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272640:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763638.597080:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272768:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763638.597103:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763638.597114:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763638/real 1787763638] req@ffff9ac451c50f00 x1874603274272768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763638.597124:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274272768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763638.597142:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763638.597145:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763638.597148:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274272768:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763639.172789:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763639.620635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763640.196829:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763640.644688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763641.220664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763641.668595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763642.244774:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763642.692702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763643.268746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763643.716656:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763643.716666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763643.716670:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763643.716694:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2688 00000100:00080000:2.0:1787763643.716705:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763643.716712:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763643.716727:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787763643.716748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274272896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763643.716771:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763643.716774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763643.716775:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763643.716784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2688 00000100:00080000:2.0:1787763643.716787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763643.716791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763643.716800:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763643.716812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf840 x1874603274273024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763643.716814:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272896:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763643.716910:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763643.716959:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763643/real 1787763643] req@ffff9ac4523bda40 x1874603274272896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763643.716983:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274272896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763643.716996:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763643.717001:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763643.717006:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274272896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763643.717036:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273024:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763643.717063:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763643.717100:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763643/real 1787763643] req@ffff9ac4523bf840 x1874603274273024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763643.717129:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf840 x1874603274273024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763643.717140:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763643.717144:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763643.717149:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273024:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763644.292723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763644.740673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763645.316767:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763645.764609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763646.340794:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763646.788630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763647.364700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763647.812641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763648.388779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763648.836694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763648.836729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763648.836736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763648.836741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763648.836753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2693 00000100:00080000:1.0:1787763648.836767:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763648.836777:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763648.836830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274273152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763648.836905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763648.836910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763648.836913:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763648.836918:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2693 00000100:00080000:1.0:1787763648.836924:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763648.836930:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763648.836945:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763648.836962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274273280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763648.836977:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273152:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763648.837077:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763648.837157:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763648/real 1787763648] req@ffff9ac447840000 x1874603274273152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763648.837192:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274273152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763648.837226:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763648.837230:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763648.837234:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273152:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763648.837259:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273280:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763648.837279:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763648.837289:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763648/real 1787763648] req@ffff9ac447842580 x1874603274273280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763648.837326:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274273280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763648.837337:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763648.837340:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763648.837345:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273280:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763649.412698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763649.860713:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763650.436700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763650.884670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763651.460692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763651.908639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763652.484674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763652.932671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763653.508678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763653.956660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763653.956711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763653.956725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763653.956734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763653.956763:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2698 00000100:00080000:1.0:1787763653.956782:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763653.956796:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763653.956851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274273408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763653.956904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763653.956912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763653.956916:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763653.956926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2698 00000100:00080000:1.0:1787763653.956950:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763653.956960:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273408:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763653.956966:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763653.956991:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763653.957023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274273536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763653.957069:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763653.957131:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763653/real 1787763653] req@ffff9ac447843840 x1874603274273408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763653.957155:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274273408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763653.957173:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763653.957181:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763653.957189:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273408:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763653.957226:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273536:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763653.957273:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763653.957293:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763653/real 1787763653] req@ffff9ac447841a40 x1874603274273536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763653.957312:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274273536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763653.957331:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763653.957339:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763653.957348:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273536:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763654.532670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763654.980682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763655.556706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763656.004689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763656.580730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763657.028694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763657.604659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763658.052654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763658.628738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763659.076615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763659.076655:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763659.076663:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763659.076668:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763659.076678:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2703 00000100:00080000:0.0:1787763659.076690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763659.076698:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763659.076742:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274273664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763659.076769:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763659.076787:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763659.076789:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763659.076794:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2703 00000100:00080000:0.0:1787763659.076798:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763659.076802:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763659.076812:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763659.076822:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763659.076826:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274273792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763659.076904:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763659.076942:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763659/real 1787763659] req@ffff9ac451f87c00 x1874603274273664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763659.076958:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274273664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763659.076968:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763659.076972:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763659.076975:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763659.076998:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273792:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763659.077015:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763659.077024:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763659/real 1787763659] req@ffff9ac451f84b40 x1874603274273792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763659.077034:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274273792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763659.077040:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763659.077043:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763659.077046:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274273792:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763659.652688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763660.100656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763660.676672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763661.124691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763661.700707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763662.148713:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763662.724676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763663.172600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763663.748728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763664.196664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763664.196675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763664.196684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763664.196689:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763664.196700:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2708 00000100:00080000:1.0:1787763664.196711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763664.196718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763664.196761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274273920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763664.196789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763664.196793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763664.196796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763664.196802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2708 00000100:00080000:1.0:1787763664.196808:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763664.196813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763664.196826:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763664.196833:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273920:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763664.196861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274274048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763664.196982:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274048:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763664.772719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763665.220609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763665.796718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763666.244649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763666.820706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763667.268608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763667.268818:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763667.268861:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763667.268879:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763667.268915:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787763667.268968:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763664/real 1787763667] req@ffff9ac4478412c0 x1874603274274048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763667.268999:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274274048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763667.269015:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763667.269021:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763667.269026:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274048:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787763667.269036:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763664/real 1787763667] req@ffff9ac447841e00 x1874603274273920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763667.269051:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274273920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763667.269060:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763667.269064:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763667.269068:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274273920:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787763667.844661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763668.292684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763668.868743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763669.316657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763669.316707:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763669.316722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763669.316747:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763669.316757:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2714 00000100:00080000:0.0:1787763669.316767:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763669.316774:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763669.316824:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274274176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763669.316868:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763669.316872:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763669.316874:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763669.316878:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2714 00000100:00080000:0.0:1787763669.316881:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763669.316919:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763669.316931:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763669.316947:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274274304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763669.316985:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274176:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763669.317059:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763669.317104:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763669/real 1787763669] req@ffff9ac451f870c0 x1874603274274176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763669.317139:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274274176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763669.317151:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763669.317156:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763669.317160:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274176:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763669.317190:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274304:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763669.317218:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763669.317234:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763669/real 1787763669] req@ffff9ac451f861c0 x1874603274274304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763669.317248:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274274304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763669.317259:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763669.317263:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763669.317267:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274304:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763669.892727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763670.340646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763670.916670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763671.364618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763671.940700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763672.388607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763672.964712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763673.412685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763673.988674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763674.436675:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763674.436675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763674.436687:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763674.436693:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763674.436702:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2719 00000100:00080000:0.0:1787763674.436715:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763674.436723:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763674.436782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274274432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763674.436838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763674.436843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763674.436846:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763674.436851:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2719 00000100:00080000:0.0:1787763674.436856:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763674.436862:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763674.436876:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763674.436908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274432:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763674.436918:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274274560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763674.437016:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763674.437087:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763674/real 1787763674] req@ffff9ac451f84780 x1874603274274432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763674.437134:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274274432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763674.437152:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763674.437159:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763674.437166:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274432:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763674.437196:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274560:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763674.437226:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763674.437240:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763674/real 1787763674] req@ffff9ac451f85680 x1874603274274560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763674.437253:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274274560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763674.437262:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763674.437265:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763674.437269:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274560:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763675.012742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763675.460662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763676.037039:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763676.484696:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763677.061401:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763677.508656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763678.084758:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763678.532661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763679.108773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763679.556649:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763679.556660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787763679.556672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763679.556696:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763679.556710:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2724 00000100:00080000:0.0:1787763679.556723:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763679.556730:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763679.556769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274274688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763679.556797:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763679.556801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763679.556802:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763679.556806:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2724 00000100:00080000:0.0:1787763679.556808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763679.556810:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763679.556820:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763679.556831:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274274816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763679.556857:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274688:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763679.556929:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763679.556981:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763679/real 1787763679] req@ffff9ac451f843c0 x1874603274274688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763679.557007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274274688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763679.557016:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763679.557020:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763679.557023:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763679.557029:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274816:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763679.557050:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763679.557060:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763679/real 1787763679] req@ffff9ac451f85680 x1874603274274816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763679.557068:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274274816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763679.557074:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763679.557077:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763679.557079:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274274816:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763680.132732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763680.580631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763681.156756:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763681.604673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763682.180748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763682.628653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763683.204752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763683.652636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763684.228747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763684.676694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763684.676710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763684.676723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763684.676731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763684.676745:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2729 00000100:00080000:1.0:1787763684.676759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763684.676769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763684.676814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274274944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763684.676853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763684.676859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763684.676861:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763684.676868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2729 00000100:00080000:1.0:1787763684.676873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763684.676890:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763684.676899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274944:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763684.676904:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763684.676932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274275072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763684.677000:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763684.677054:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763684/real 1787763684] req@ffff9ac447843480 x1874603274274944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763684.677085:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274274944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763684.677096:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763684.677100:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763684.677103:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274274944:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763684.677131:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275072:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763684.677150:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763684.677159:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763684/real 1787763684] req@ffff9ac447841e00 x1874603274275072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763684.677168:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274275072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763684.677174:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763684.677177:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763684.677179:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275072:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763685.252768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763685.700626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763686.276663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763686.724602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763687.300656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763687.748665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763688.324717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763688.772734:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763689.348730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763689.796686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763689.796692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763689.796697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763689.796703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763689.796717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2734 00000100:00080000:1.0:1787763689.796729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763689.796738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763689.796779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274275200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763689.796806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763689.796810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763689.796811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763689.796817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2734 00000100:00080000:1.0:1787763689.796823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763689.796831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763689.796844:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763689.796847:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275200:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763689.796864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274275328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763689.796931:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763689.796977:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763689/real 1787763689] req@ffff9ac4478412c0 x1874603274275200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763689.797021:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274275200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763689.797036:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763689.797044:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763689.797049:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275200:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763689.797080:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275328:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763689.797122:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763689.797136:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763689/real 1787763689] req@ffff9ac447841a40 x1874603274275328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763689.797149:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274275328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763689.797156:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763689.797160:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763689.797162:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275328:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763690.372791:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763690.820654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763691.396711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763691.844736:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763692.420742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763692.868677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763693.444829:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763693.892663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763694.468747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763694.916713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763694.916715:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763694.916737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763694.916744:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763694.916757:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2739 00000100:00080000:2.0:1787763694.916803:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763694.916818:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763694.916875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274275456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763694.916928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763694.916936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763694.916940:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763694.916949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2739 00000100:00080000:2.0:1787763694.916957:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763694.916965:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763694.916981:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763694.917004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274275584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763694.917019:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275456:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763694.917132:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763694.917187:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763694/real 1787763694] req@ffff9ac4523bda40 x1874603274275456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763694.917208:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274275456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763694.917219:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763694.917224:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763694.917228:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763694.917259:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275584:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763694.917279:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763694.917310:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763694/real 1787763694] req@ffff9ac4523bc3c0 x1874603274275584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763694.917322:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274275584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763694.917344:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763694.917348:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763694.917350:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275584:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763695.492738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763695.940708:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763696.516809:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763696.964643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763697.540783:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763697.988656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763698.564790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763699.012621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763699.588775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763700.036605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763700.036734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763700.036748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763700.036754:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763700.036766:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2744 00000100:00080000:1.0:1787763700.036780:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763700.036790:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763700.036860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274275712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763700.036899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763700.036904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763700.036907:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763700.036926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2744 00000100:00080000:1.0:1787763700.036933:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763700.036948:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763700.036948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275712:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763700.036965:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763700.036989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274275840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763700.037060:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763700.037112:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763700/real 1787763700] req@ffff9ac447843840 x1874603274275712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763700.037129:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274275712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763700.037139:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763700.037143:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763700.037146:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275712:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763700.037171:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275840:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763700.037188:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763700.037213:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763700/real 1787763700] req@ffff9ac447842580 x1874603274275840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763700.037222:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274275840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763700.037229:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763700.037231:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763700.037234:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274275840:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763700.612806:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763701.060617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763701.636758:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763702.084615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763702.660768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763703.108613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763703.684715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763704.132647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763704.708773:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763705.156657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763705.156729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763705.156742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763705.156749:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763705.156766:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2749 00000100:00080000:2.0:1787763705.156783:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763705.156796:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763705.156849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274275968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763705.156900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763705.156908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763705.156936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763705.156948:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2749 00000100:00080000:2.0:1787763705.156962:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787763705.156968:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275968:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787763705.156979:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763705.157000:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763705.157028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274276096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787763705.157070:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763705.157143:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763705/real 1787763705] req@ffff9ac4523be940 x1874603274275968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763705.157181:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274275968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763705.157202:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763705.157208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763705.157253:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274275968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763705.157288:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276096:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763705.157325:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763705.157343:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763705/real 1787763705] req@ffff9ac4523bc000 x1874603274276096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763705.157360:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274276096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763705.157371:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763705.157376:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763705.157380:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276096:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763705.732800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763706.180650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763706.756781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763707.204614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763707.780833:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763708.228669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763708.804744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763709.252638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763709.828800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763710.276641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763710.276683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763710.276693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763710.276698:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763710.276710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2755 00000100:00080000:2.0:1787763710.276725:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763710.276733:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763710.276795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274276224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763710.276822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763710.276825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763710.276827:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763710.276831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2755 00000100:00080000:2.0:1787763710.276833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763710.276836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763710.276846:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763710.276859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274276352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787763710.276903:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276224:192.168.203.123@tcp:250:kworker.0 00000800:00000100:3.0:1787763710.276983:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763710.277046:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763710/real 1787763710] req@ffff9ac4523bd680 x1874603274276224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763710.277092:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274276224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763710.277104:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763710.277108:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763710.277113:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276224:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763710.277142:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276352:192.168.203.123@tcp:38:kworker.0 00000800:00000100:3.0:1787763710.277168:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:3.0:1787763710.277203:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763710/real 1787763710] req@ffff9ac4523bc780 x1874603274276352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787763710.277233:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274276352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787763710.277241:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787763710.277244:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787763710.277247:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276352:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763710.852699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763711.300609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763711.876744:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763712.324598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763712.900721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763713.348641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763713.924747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763714.372618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763714.948703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763715.396588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763715.396695:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763715.396701:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763715.396704:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763715.396711:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2760 00000100:00080000:3.0:1787763715.396721:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763715.396727:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763715.396759:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274276480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763715.396785:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763715.396788:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763715.396789:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763715.396792:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2760 00000100:00080000:3.0:1787763715.396794:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763715.396797:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763715.396805:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763715.396815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274276608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763715.396863:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276480:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763715.396968:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763715.397025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763715/real 1787763715] req@ffff9ac451c50f00 x1874603274276480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763715.397048:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274276480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763715.397062:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763715.397068:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763715.397072:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276480:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763715.397081:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276608:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763715.397105:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763715.397117:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763715/real 1787763715] req@ffff9ac451c52580 x1874603274276608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763715.397131:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274276608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763715.397140:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763715.397173:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763715.397180:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276608:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763715.972741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763716.420612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763716.996726:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763717.444610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763718.020683:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763718.468655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763719.044790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763719.492653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763720.068795:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763720.516630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763720.516631:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763720.516641:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763720.516647:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763720.516657:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2765 00000100:00080000:0.0:1787763720.516671:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763720.516679:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763720.516728:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274276736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763720.516779:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763720.516784:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763720.516791:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763720.516797:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2765 00000100:00080000:0.0:1787763720.516802:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763720.516808:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763720.516820:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763720.516836:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274276864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763720.516838:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276736:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763720.516923:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763720.516984:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763720/real 1787763720] req@ffff9ac451f84780 x1874603274276736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763720.516999:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274276736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763720.517008:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763720.517011:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763720.517014:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276736:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763720.517037:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276864:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763720.517060:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763720.517076:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763720/real 1787763720] req@ffff9ac451f861c0 x1874603274276864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763720.517089:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274276864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763720.517098:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763720.517101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763720.517105:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274276864:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763721.092681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763721.540599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763722.116768:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763722.564662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763723.140717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763723.588616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763724.164723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787763724.612641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763725.188763:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763725.636686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763725.636700:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763725.636706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763725.636721:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2770 00000100:00080000:3.0:1787763725.636738:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763725.636752:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763725.636800:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787763725.636801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274276992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763725.636851:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763725.636855:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763725.636858:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763725.636864:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2770 00000100:00080000:3.0:1787763725.636871:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763725.636877:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763725.636886:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276992:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787763725.636890:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763725.636912:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274277120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763725.636966:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763725.637028:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763725/real 1787763725] req@ffff9ac451c503c0 x1874603274276992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763725.637046:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274276992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763725.637058:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763725.637062:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763725.637066:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274276992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763725.637153:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277120:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763725.637203:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763725.637241:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763725/real 1787763725] req@ffff9ac451c50780 x1874603274277120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763725.637262:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274277120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763725.637297:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763725.637304:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763725.637309:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277120:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763726.212761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763726.660663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763727.236744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763727.684679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763728.260674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763728.708626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763729.284799:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763729.732644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763730.308726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763730.756653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763730.756730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763730.756740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763730.756746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763730.756758:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2775 00000100:00080000:2.0:1787763730.756771:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763730.756780:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763730.756836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274277248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763730.756888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763730.756891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763730.756898:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763730.756920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2775 00000100:00080000:2.0:1787763730.756924:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763730.756932:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763730.756945:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763730.756969:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274277248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763730.756970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274277376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763730.757154:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277376:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763731.332818:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763731.780662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763732.356800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763732.804615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763733.380813:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763733.828682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787763733.828897:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787763733.828948:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787763733.828969:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787763733.829010:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763733.829173:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763730/real 1787763733] req@ffff9ac4523bcf00 x1874603274277376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763733.829229:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274277376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763733.829252:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763733.829262:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763733.829270:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277376:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787763733.829317:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763730/real 1787763733] req@ffff9ac4523be580 x1874603274277248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763733.829371:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274277248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763733.829387:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763733.829394:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763733.829402:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274277248:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763734.404648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763734.852703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763735.428733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763735.876758:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763735.876763:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763735.876774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763735.876782:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763735.876794:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2780 00000100:00080000:3.0:1787763735.876809:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763735.876819:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763735.876862:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274277504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763735.876901:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763735.876906:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763735.876908:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763735.876913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2780 00000100:00080000:3.0:1787763735.876918:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763735.876922:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763735.876934:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763735.876967:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274277632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763735.877061:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274277504:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763735.877191:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763735.877262:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763735/real 1787763735] req@ffff9ac451c52940 x1874603274277504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763735.877288:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274277504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763735.877308:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763735.877314:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763735.877320:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274277504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763735.877331:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277632:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763735.877357:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763735.877373:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763735/real 1787763735] req@ffff9ac451c50b40 x1874603274277632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763735.877386:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274277632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763735.877420:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763735.877424:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763735.877429:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277632:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763736.452721:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763736.900785:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763737.476725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763737.924617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763738.501633:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763738.948587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763739.524693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763739.972673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763740.549604:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763740.996609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763740.996630:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763740.996636:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763740.996639:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763740.996647:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2785 00000100:00080000:3.0:1787763740.996656:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763740.996669:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763740.996700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274277760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763740.996721:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763740.996723:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763740.996724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763740.996727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2785 00000100:00080000:3.0:1787763740.996730:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763740.996732:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763740.996740:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763740.996751:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274277760:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763740.996751:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274277888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763740.996838:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763740.996902:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763740/real 1787763740] req@ffff9ac451c530c0 x1874603274277760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763740.996926:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274277760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763740.996941:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763740.996946:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763740.996952:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274277760:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763740.997020:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277888:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763740.997042:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763740.997052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763740/real 1787763740] req@ffff9ac451c52d00 x1874603274277888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763740.997085:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274277888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763740.997097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763740.997121:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763740.997126:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274277888:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763741.572878:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763742.020797:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763742.596835:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763743.044589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763743.620855:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763744.068627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763744.644805:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763745.092650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763745.668674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763746.116631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763746.116664:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763746.116670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763746.116676:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763746.116687:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2790 00000100:00080000:0.0:1787763746.116701:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763746.116709:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763746.116757:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274278016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763746.116788:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763746.116791:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763746.116793:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763746.116796:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2790 00000100:00080000:0.0:1787763746.116799:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763746.116802:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763746.116819:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278016:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763746.116820:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763746.116840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274278144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763746.116903:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763746.116967:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763746/real 1787763746] req@ffff9ac451f870c0 x1874603274278016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763746.117007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274278016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763746.117017:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763746.117021:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763746.117025:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763746.117049:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278144:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763746.117069:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763746.117079:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763746/real 1787763746] req@ffff9ac451f84000 x1874603274278144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763746.117087:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274278144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763746.117093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763746.117095:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763746.117097:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278144:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763746.692765:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763747.140632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763747.716771:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763748.164631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763748.740815:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763749.188643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763749.764728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763750.212625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763750.788690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763751.236644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763751.236644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763751.236655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763751.236681:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763751.236692:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2795 00000100:00080000:0.0:1787763751.236717:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763751.236726:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763751.236769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274278272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763751.236791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763751.236794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763751.236796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763751.236799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2795 00000100:00080000:0.0:1787763751.236802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763751.236805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763751.236815:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763751.236823:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763751.236827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274278400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763751.236923:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763751.236980:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763751/real 1787763751] req@ffff9ac451f84b40 x1874603274278272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763751.237002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274278272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763751.237012:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763751.237015:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763751.237018:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763751.237041:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278400:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763751.237061:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763751.237070:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763751/real 1787763751] req@ffff9ac451f87c00 x1874603274278400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763751.237078:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274278400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763751.237084:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763751.237087:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763751.237089:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278400:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763751.812675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763752.260626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763752.836851:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763753.284689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763753.860806:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763754.308715:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763754.884789:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763755.332679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763755.908765:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763756.356681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763756.356748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763756.356766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763756.356773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763756.356787:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2801 00000100:00080000:1.0:1787763756.356802:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763756.356811:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763756.356909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274278528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763756.356950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763756.356955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763756.356958:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763756.356965:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2801 00000100:00080000:1.0:1787763756.356972:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763756.356980:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763756.356993:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763756.357000:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763756.357017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274278656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763756.357106:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763756.357156:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763756/real 1787763756] req@ffff9ac447840000 x1874603274278528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763756.357191:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274278528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763756.357203:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763756.357208:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763756.357212:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763756.357241:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278656:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763756.357263:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763756.357274:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763756/real 1787763756] req@ffff9ac447840f00 x1874603274278656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763756.357284:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274278656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763756.357292:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763756.357294:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763756.357297:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278656:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763756.932790:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763757.380655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763757.956723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763758.404665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763758.980674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763759.428609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763760.004701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763760.452637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763761.028725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763761.476687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763761.476700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763761.476710:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763761.476712:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763761.476727:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2806 00000100:00080000:1.0:1787763761.476746:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763761.476760:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763761.476822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274278784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763761.476895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763761.476902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763761.476906:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763761.476938:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2806 00000100:00080000:1.0:1787763761.476948:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763761.476959:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278784:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763761.476963:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763761.476990:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763761.477023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274278912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763761.477090:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763761.477164:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763761/real 1787763761] req@ffff9ac447840780 x1874603274278784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763761.477214:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274278784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763761.477229:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763761.477235:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763761.477240:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274278784:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763761.477291:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278912:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763761.477312:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763761.477323:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763761/real 1787763761] req@ffff9ac447841680 x1874603274278912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763761.477332:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274278912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763761.477338:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763761.477341:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763761.477343:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274278912:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763762.052686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763762.500613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763763.076779:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763763.524667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763764.100747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763764.548667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763765.124799:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763765.572690:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763766.148858:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763766.596676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763766.596738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763766.596750:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763766.596756:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763766.596768:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2811 00000100:00080000:0.0:1787763766.596783:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763766.596813:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763766.596866:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274279040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763766.596907:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763766.596932:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763766.596936:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763766.596943:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2811 00000100:00100000:2.0:1787763766.596952:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279040:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787763766.596956:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763766.596976:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763766.596990:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763766.597012:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274279168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763766.597081:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763766.597142:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763766/real 1787763766] req@ffff9ac451f87c00 x1874603274279040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763766.597174:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274279040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763766.597185:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763766.597189:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763766.597192:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279040:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763766.597221:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279168:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763766.597241:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763766.597251:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763766/real 1787763766] req@ffff9ac451f84b40 x1874603274279168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763766.597260:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274279168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763766.597266:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763766.597268:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763766.597270:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279168:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763767.172839:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763767.620616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763768.196739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763768.644691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763769.220713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763769.669595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763770.244813:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763770.692694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763771.268675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763771.716620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763771.716675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763771.716685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763771.716690:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763771.716700:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2816 00000100:00080000:1.0:1787763771.716712:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763771.716719:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763771.716782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274279296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763771.716833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763771.716837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763771.716840:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763771.716846:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2816 00000100:00080000:1.0:1787763771.716851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763771.716857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763771.716877:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279296:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763771.716897:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763771.716922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274279424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763771.716978:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763771.717035:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763771/real 1787763771] req@ffff9ac4478403c0 x1874603274279296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763771.717071:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274279296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763771.717084:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763771.717089:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763771.717093:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763771.717119:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279424:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763771.717139:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763771.717148:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763771/real 1787763771] req@ffff9ac4478430c0 x1874603274279424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763771.717157:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274279424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763771.717163:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763771.717166:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763771.717168:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279424:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763772.292748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763772.740659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763773.316772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763773.764661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763774.340814:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763774.788642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763775.364681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763775.812651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763776.388641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763776.836684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763776.836697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763776.836703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763776.836716:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2821 00000100:00100000:2.0:1787763776.836745:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763776.836757:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763776.836771:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763776.836821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274279552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763776.836859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763776.836863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763776.836866:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763776.836872:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2821 00000100:00080000:1.0:1787763776.836876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763776.836882:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763776.836896:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763776.836916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279552:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763776.836917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274279680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763776.837017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763776.837081:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763776/real 1787763776] req@ffff9ac4478421c0 x1874603274279552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763776.837130:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274279552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763776.837144:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763776.837150:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763776.837155:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279552:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763776.837190:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279680:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763776.837235:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763776.837260:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763776/real 1787763776] req@ffff9ac447842d00 x1874603274279680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763776.837287:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274279680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763776.837305:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763776.837313:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763776.837318:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279680:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763777.412631:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763777.860647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763778.436674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763778.884604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763779.460692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763779.908679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763780.484663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763780.932678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763781.508785:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763781.956676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763781.956708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763781.956714:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763781.956733:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763781.956745:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2826 00000100:00080000:0.0:1787763781.956762:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763781.956772:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763781.956823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274279808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763781.956854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763781.956858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763781.956860:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763781.956864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2826 00000100:00080000:0.0:1787763781.956867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763781.956870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763781.956880:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763781.956891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274279936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763781.956914:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279808:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763781.957027:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763781.957111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763781/real 1787763781] req@ffff9ac451f84000 x1874603274279808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763781.957158:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274279808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763781.957173:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763781.957179:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763781.957184:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274279808:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763781.957215:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279936:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763781.957249:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763781.957265:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763781/real 1787763781] req@ffff9ac451f870c0 x1874603274279936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763781.957281:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274279936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763781.957292:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763781.957296:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763781.957300:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274279936:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763782.532772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763782.980657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763783.556778:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763784.004669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763784.580798:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763785.028645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763785.604769:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763786.052717:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763786.628681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763787.076687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763787.076796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763787.076810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763787.076819:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763787.076835:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2831 00000100:00080000:1.0:1787763787.076856:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763787.076866:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763787.076924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274280064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763787.076975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763787.076981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763787.076984:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763787.076990:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2831 00000100:00080000:1.0:1787763787.076997:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763787.077004:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763787.077018:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763787.077037:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763787.077044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274280192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763787.077130:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763787.077202:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763787/real 1787763787] req@ffff9ac447840b40 x1874603274280064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763787.077282:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274280064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763787.077298:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763787.077305:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763787.077310:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763787.077343:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280192:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763787.077378:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763787.077395:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763787/real 1787763787] req@ffff9ac447842940 x1874603274280192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763787.077411:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274280192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763787.077422:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763787.077426:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763787.077430:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280192:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763787.652713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763788.100672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763788.676682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763789.124616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763789.700732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763790.148665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763790.724691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763791.172636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763791.748707:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763792.196687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763792.196715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763792.196727:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763792.196734:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763792.196748:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2836 00000100:00080000:0.0:1787763792.196762:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763792.196773:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763792.196837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274280320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763792.196888:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763792.196894:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763792.196896:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763792.196902:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2836 00000100:00080000:0.0:1787763792.196907:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763792.196913:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763792.196926:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763792.196942:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274280448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763792.196948:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280320:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763792.197033:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763792.197087:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763792/real 1787763792] req@ffff9ac451f861c0 x1874603274280320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763792.197133:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274280320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763792.197143:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763792.197146:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763792.197150:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763792.197175:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280448:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763792.197193:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763792.197202:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763792/real 1787763792] req@ffff9ac451f84780 x1874603274280448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763792.197211:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274280448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763792.197217:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763792.197219:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763792.197221:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280448:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763792.772658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763793.220632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763793.796804:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763794.244696:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763794.820702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763795.268619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763795.844718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763796.292668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763796.868704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763797.316650:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763797.316659:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763797.316664:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763797.316674:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2842 00000100:00080000:3.0:1787763797.316688:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763797.316695:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763797.316743:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787763797.316747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274280576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763797.316788:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763797.316792:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763797.316794:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763797.316799:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2842 00000100:00080000:3.0:1787763797.316804:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763797.316808:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763797.316819:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763797.316832:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763797.316833:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274280704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763797.316938:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280704:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763797.892730:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763798.340714:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763798.916719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763799.364819:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763799.940769:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763800.388895:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:3.0:1787763800.388947:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:3.0:1787763800.389000:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787763800.389029:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:3.0:1787763800.389085:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763800.389183:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763797/real 1787763800] req@ffff9ac451c530c0 x1874603274280704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763800.389242:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274280704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763800.389259:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763800.389266:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763800.389271:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280704:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787763800.389279:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763797/real 1787763800] req@ffff9ac451c52d00 x1874603274280576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763800.389293:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274280576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763800.389301:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763800.389304:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763800.389308:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280576:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787763800.964783:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763801.412741:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763801.988745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763802.436606:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763802.436615:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763802.436618:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763802.436627:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2847 00000100:00080000:3.0:1787763802.436637:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763802.436645:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763802.436680:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274280832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763802.436715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763802.436718:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763802.436720:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763802.436724:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2847 00000100:00080000:3.0:1787763802.436727:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763802.436730:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763802.436739:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763802.436753:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274280960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763802.436761:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763802.436812:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000800:00000100:0.0:1787763802.436828:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763802.436874:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763802/real 1787763802] req@ffff9ac451c50b40 x1874603274280832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763802.436889:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274280832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763802.436897:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763802.436901:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763802.436905:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274280832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763802.436942:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280960:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763802.436969:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763802.436983:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763802/real 1787763802] req@ffff9ac451c52940 x1874603274280960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763802.436996:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274280960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763802.437006:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763802.437009:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763802.437013:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274280960:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763803.012760:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763803.460866:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763804.036677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763804.484619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763805.060951:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763805.508611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763806.084690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763806.532603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763807.108643:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763807.556638:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763807.556651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763807.556657:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763807.556661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763807.556672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2852 00000100:00080000:3.0:1787763807.556685:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763807.556698:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763807.556757:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274281088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763807.556798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763807.556803:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763807.556824:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763807.556832:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2852 00000100:00080000:3.0:1787763807.556857:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763807.556866:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763807.556886:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763807.556894:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763807.556923:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274281216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763807.556997:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763807.557052:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763807/real 1787763807] req@ffff9ac451c50780 x1874603274281088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763807.557078:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274281088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763807.557093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763807.557099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763807.557104:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763807.557137:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281216:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763807.557180:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763807.557193:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763807/real 1787763807] req@ffff9ac451c503c0 x1874603274281216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763807.557218:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274281216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763807.557228:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763807.557231:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763807.557235:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281216:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763808.132719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763808.580710:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763809.156847:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763809.604682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763810.180808:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763810.628650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763811.204720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763811.652617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763812.228755:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763812.676653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763812.676660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763812.676676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763812.676684:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763812.676712:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2857 00000100:00080000:1.0:1787763812.676729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763812.676741:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763812.676802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274281344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763812.676843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763812.676848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763812.676851:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763812.676858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2857 00000100:00080000:1.0:1787763812.676864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763812.676871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763812.676886:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763812.676904:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763812.676910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274281472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763812.677001:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763812.677066:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763812/real 1787763812] req@ffff9ac447843c00 x1874603274281344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763812.677092:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274281344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763812.677119:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763812.677125:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763812.677132:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763812.677165:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281472:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763812.677201:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763812.677214:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763812/real 1787763812] req@ffff9ac447842940 x1874603274281472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763812.677225:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274281472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763812.677272:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763812.677276:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763812.677279:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763813.252792:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763813.700665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763814.276800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763814.724668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763815.300744:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763815.748690:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763816.324799:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763816.772650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763817.348750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763817.796630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763817.796641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763817.796648:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763817.796659:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2862 00000100:00100000:2.0:1787763817.796668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763817.796673:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763817.796688:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763817.796757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274281600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763817.796785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763817.796788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763817.796790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763817.796794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2862 00000100:00080000:1.0:1787763817.796797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763817.796800:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763817.796812:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763817.796822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274281728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763817.796850:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281600:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763817.796930:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763817.796984:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763817/real 1787763817] req@ffff9ac447840b40 x1874603274281600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763817.797016:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274281600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763817.797034:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763817.797040:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763817.797044:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763817.797053:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281728:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763817.797082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763817.797097:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763817/real 1787763817] req@ffff9ac447842d00 x1874603274281728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763817.797109:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274281728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763817.797119:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763817.797122:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763817.797126:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281728:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763818.372842:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763818.820728:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763819.396774:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763819.844679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763820.420742:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763820.868645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763821.444781:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763821.892633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763822.468703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763822.916645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763822.916710:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763822.916719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763822.916725:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763822.916736:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2867 00000100:00080000:0.0:1787763822.916750:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763822.916759:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763822.916804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274281856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763822.916844:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763822.916849:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763822.916851:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763822.916857:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2867 00000100:00080000:0.0:1787763822.916863:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763822.916871:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763822.916888:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763822.916897:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763822.916918:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274281984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763822.916991:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763822.917054:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763822/real 1787763822] req@ffff9ac451f85680 x1874603274281856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763822.917080:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274281856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763822.917093:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763822.917099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763822.917128:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274281856:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763822.917169:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281984:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763822.917246:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763822.917266:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763822/real 1787763822] req@ffff9ac451f843c0 x1874603274281984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763822.917283:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274281984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763822.917294:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763822.917299:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763822.917303:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274281984:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763823.492731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763823.940661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763824.516665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763824.964669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763825.540659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763825.988610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763826.564664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763827.012700:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763827.588662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763828.036594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763828.036613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763828.036619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763828.036624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763828.036634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2872 00000100:00080000:1.0:1787763828.036645:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763828.036653:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763828.036685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274282112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763828.036706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763828.036709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763828.036711:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763828.036714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2872 00000100:00080000:1.0:1787763828.036717:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787763828.036735:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282112:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763828.036755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763828.036766:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763828.036784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274282240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763828.036794:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763828.036836:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763828/real 1787763828] req@ffff9ac4478421c0 x1874603274282112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763828.036855:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274282112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763828.036866:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763828.036869:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763828.036879:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282112:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763828.036906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274282240:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763828.036926:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763828.036936:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763828/real 1787763828] req@ffff9ac4478430c0 x1874603274282240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763828.036944:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274282240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763828.036950:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763828.036952:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763828.036955:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274282240:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763828.612703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763829.060622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763829.636642:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763830.084653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763830.660789:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763831.108607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763831.684641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763832.132604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763832.708656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763833.156702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763833.156722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763833.156733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763833.156739:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763833.156752:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2877 00000100:00080000:0.0:1787763833.156781:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763833.156788:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763833.156818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274282368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763833.156856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763833.156860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763833.156862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763833.156867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2877 00000100:00080000:0.0:1787763833.156873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763833.156879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763833.156888:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282368:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763833.156889:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763833.156906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274282496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763833.156949:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763833.156981:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763833/real 1787763833] req@ffff9ac451f843c0 x1874603274282368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763833.156993:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274282368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763833.157001:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763833.157004:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763833.157013:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763833.157030:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274282496:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763833.157048:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763833.157056:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763833/real 1787763833] req@ffff9ac451f85680 x1874603274282496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763833.157064:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274282496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763833.157070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763833.157073:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763833.157075:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274282496:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763833.732653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763834.180611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763834.756676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763835.204621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763835.780714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763836.228614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763836.804690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763837.252639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763837.828666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763838.276615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763838.276632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763838.276638:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763838.276642:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763838.276649:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2883 00000100:00080000:3.0:1787763838.276659:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763838.276664:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763838.276695:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274282624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763838.276715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763838.276718:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763838.276719:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763838.276722:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2883 00000100:00080000:3.0:1787763838.276725:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763838.276727:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763838.276735:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763838.276746:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763838.276746:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274282752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763838.276806:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763838.276837:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763838/real 1787763838] req@ffff9ac451c52580 x1874603274282624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763838.276848:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274282624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763838.276856:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763838.276859:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763838.276874:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763838.276896:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274282752:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763838.276920:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763838.276934:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763838/real 1787763838] req@ffff9ac451c50f00 x1874603274282752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763838.276947:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274282752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763838.276957:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763838.276959:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763838.276961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274282752:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763838.852678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763839.300633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763839.876700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763840.324608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763840.900743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763841.348619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763841.924678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763842.372642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763842.948663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763843.396602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763843.396611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763843.396617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763843.396621:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763843.396630:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2888 00000100:00080000:1.0:1787763843.396639:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763843.396645:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763843.396682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274282880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763843.396708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763843.396712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763843.396713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763843.396718:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2888 00000100:00080000:1.0:1787763843.396723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763843.396728:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763843.396735:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282880:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763843.396738:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763843.396755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274283008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763843.396803:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763843.396834:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763843/real 1787763843] req@ffff9ac4478403c0 x1874603274282880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763843.396850:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274282880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763843.396858:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763843.396861:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763843.396876:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274282880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763843.396901:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283008:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763843.396919:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763843.396928:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763843/real 1787763843] req@ffff9ac447841680 x1874603274283008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763843.396936:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274283008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763843.396942:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763843.396944:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763843.396946:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283008:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763843.972690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763844.420631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763844.996650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763845.444606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763846.020654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763846.468610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763847.044688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763847.492590:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763848.068687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763848.516594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763848.516603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763848.516609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763848.516625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763848.516633:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2893 00000100:00080000:1.0:1787763848.516643:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763848.516650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763848.516680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274283136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763848.516700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763848.516703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763848.516704:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763848.516707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2893 00000100:00080000:1.0:1787763848.516710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763848.516713:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763848.516723:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763848.516731:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787763848.516742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274283264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763848.516788:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763848.516817:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763848/real 1787763848] req@ffff9ac447840780 x1874603274283136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763848.516829:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274283136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763848.516837:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763848.516840:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763848.516849:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763848.516867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283264:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763848.516884:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763848.516893:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763848/real 1787763848] req@ffff9ac447840f00 x1874603274283264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763848.516901:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274283264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763848.516907:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763848.516909:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763848.516911:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283264:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763849.092663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763849.540588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763850.116672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763850.564634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763851.140663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763851.588603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763852.164708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763852.612643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763853.188654:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763853.636589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763853.636601:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763853.636609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763853.636613:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763853.636624:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2898 00000100:00080000:1.0:1787763853.636635:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763853.636643:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763853.636680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274283392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763853.636709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763853.636736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763853.636739:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763853.636745:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283392:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787763853.636748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2898 00000100:00080000:1.0:1787763853.636758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763853.636763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763853.636774:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763853.636792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274283520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763853.636816:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763853.636854:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763853/real 1787763853] req@ffff9ac447840000 x1874603274283392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763853.636874:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274283392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763853.636884:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763853.636888:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763853.636900:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763853.636923:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283520:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763853.636950:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763853.636964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763853/real 1787763853] req@ffff9ac447842580 x1874603274283520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763853.636977:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274283520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763853.636986:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763853.636990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763853.636994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283520:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763854.212716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763854.660654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763855.236651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763855.684588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763856.260704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763856.708645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763857.284723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763857.732608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763858.308664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763858.756592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763858.756625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763858.756631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763858.756634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763858.756642:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2903 00000100:00080000:2.0:1787763858.756651:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763858.756657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763858.756688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274283648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763858.756710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763858.756713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763858.756714:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763858.756717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2903 00000100:00080000:2.0:1787763858.756719:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763858.756735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763858.756742:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283648:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787763858.756744:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763858.756756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274283776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763858.756798:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763858.756837:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763858/real 1787763858] req@ffff9ac4523bcb40 x1874603274283648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763858.756856:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274283648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763858.756864:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763858.756866:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763858.756869:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763858.756895:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283776:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763858.756915:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763858.756924:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763858/real 1787763858] req@ffff9ac4523be1c0 x1874603274283776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763858.756933:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274283776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763858.756938:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763858.756941:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763858.756943:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274283776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763859.332666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763859.780579:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763860.356681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763860.804602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763861.380676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763861.828639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763862.404692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763862.852615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763863.428691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763863.876620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763863.876623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763863.876631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763863.876635:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763863.876643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2908 00000100:00080000:2.0:1787763863.876653:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763863.876659:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763863.876692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274283904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763863.876715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763863.876718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763863.876720:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763863.876729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2908 00000100:00080000:2.0:1787763863.876732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763863.876735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763863.876745:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763863.876756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274284032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763863.876779:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763863.876867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763864.452689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763864.900627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763865.476712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763865.924674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763866.500731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763866.948647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763866.948842:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763866.948892:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763866.948911:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763866.948949:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763866.948997:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763863/real 1787763866] req@ffff9ac4523bf0c0 x1874603274283904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763866.949027:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274283904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763866.949070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763866.949077:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763866.949083:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274283904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763866.949092:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763863/real 1787763866] req@ffff9ac4523be1c0 x1874603274284032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763866.949123:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274284032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763866.949133:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763866.949137:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763866.949141:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763867.524733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763867.972634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763868.548711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763868.996595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763868.996602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763868.996608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763868.996623:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763868.996632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2913 00000100:00080000:2.0:1787763868.996642:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763868.996647:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763868.996676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274284160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763868.996697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763868.996699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763868.996701:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763868.996704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2913 00000100:00080000:2.0:1787763868.996706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763868.996709:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763868.996717:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763868.996727:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763868.996733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274284288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763868.996785:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763868.996821:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763868/real 1787763868] req@ffff9ac4523bcb40 x1874603274284160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763868.996840:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274284160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763868.996852:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763868.996870:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763868.996875:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763868.996921:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284288:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763868.996949:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763868.996964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763868/real 1787763868] req@ffff9ac4523be580 x1874603274284288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763868.996978:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274284288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763868.996987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763868.996990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763868.996994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284288:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763869.572701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763870.020592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763870.596665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763871.044597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763871.620720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763872.068617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763872.644675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763873.092589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763873.668677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763874.116607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763874.116756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763874.116765:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763874.116770:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763874.116780:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2918 00000100:00080000:3.0:1787763874.116792:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763874.116800:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763874.116852:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274284416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763874.116888:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763874.116893:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763874.116895:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763874.116901:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2918 00000100:00080000:3.0:1787763874.116907:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763874.116913:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763874.116924:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763874.116925:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763874.116945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274284544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763874.117002:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763874.117045:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763874/real 1787763874] req@ffff9ac451c50f00 x1874603274284416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763874.117071:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274284416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763874.117080:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763874.117083:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763874.117086:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763874.117121:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284544:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763874.117143:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763874.117157:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763874/real 1787763874] req@ffff9ac451c52580 x1874603274284544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763874.117171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274284544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763874.117180:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763874.117184:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763874.117188:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763874.692638:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763875.140588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763875.716664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763876.164610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763876.740741:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763877.188658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763877.764743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763878.212662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763878.788780:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763879.236598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763879.236601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763879.236609:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763879.236614:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763879.236625:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2923 00000100:00080000:3.0:1787763879.236636:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763879.236672:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763879.236717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274284672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763879.236744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763879.236747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763879.236748:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763879.236751:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2923 00000100:00080000:3.0:1787763879.236756:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763879.236760:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763879.236769:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763879.236775:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763879.236783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274284800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763879.236846:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763879.236887:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763879/real 1787763879] req@ffff9ac451c503c0 x1874603274284672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763879.236901:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274284672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763879.236910:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763879.236914:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763879.236917:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763879.236946:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284800:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763879.236964:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763879.236972:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763879/real 1787763879] req@ffff9ac451c50780 x1874603274284800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763879.236981:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274284800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763879.236987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763879.236989:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763879.236991:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274284800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763879.812673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763880.260617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763880.836700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763881.284681:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763881.860695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763882.308607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763882.884709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763883.332642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763883.908678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763884.356595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763884.356595:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763884.356605:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763884.356608:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763884.356616:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2929 00000100:00080000:3.0:1787763884.356626:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763884.356633:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763884.356664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274284928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763884.356683:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763884.356686:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763884.356699:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763884.356704:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2929 00000100:00080000:3.0:1787763884.356707:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787763884.356711:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284928:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787763884.356713:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763884.356727:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763884.356744:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274285056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763884.356773:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763884.356807:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763884/real 1787763884] req@ffff9ac451c52940 x1874603274284928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763884.356819:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274284928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763884.356827:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763884.356830:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763884.356833:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274284928:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763884.356880:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285056:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763884.356917:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763884.356927:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763884/real 1787763884] req@ffff9ac451c50b40 x1874603274285056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763884.356937:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274285056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763884.356943:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763884.356946:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763884.356948:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285056:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763884.932682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763885.380602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763885.956679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763886.404605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763886.980711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763887.428625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763888.004640:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763888.452609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763889.028662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763889.476617:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763889.476619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763889.476627:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763889.476631:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763889.476640:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2934 00000100:00080000:3.0:1787763889.476649:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763889.476654:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763889.476687:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274285184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763889.476716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763889.476720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763889.476723:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763889.476729:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2934 00000100:00080000:3.0:1787763889.476734:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763889.476739:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763889.476743:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285184:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787763889.476750:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763889.476779:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274285312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763889.476803:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763889.476843:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763889/real 1787763889] req@ffff9ac451c52d00 x1874603274285184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763889.476856:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274285184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763889.476864:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763889.476867:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763889.476869:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763889.476899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285312:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763889.476917:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763889.476926:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763889/real 1787763889] req@ffff9ac451c530c0 x1874603274285312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763889.476934:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274285312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763889.476940:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763889.476942:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763889.476961:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763890.052673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763890.500593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763891.076705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763891.524636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763892.100662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763892.548633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763893.124836:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763893.572618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763894.148680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787763894.596622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763894.596630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1787763894.596634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763894.596634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763894.596645:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2939 00000100:00080000:2.0:1787763894.596655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763894.596661:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763894.596692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274285440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763894.596712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763894.596728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763894.596730:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763894.596734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2939 00000100:00080000:2.0:1787763894.596737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787763894.596739:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285440:192.168.203.123@tcp:250:kworker.0 00000100:00080000:2.0:1787763894.596743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763894.596754:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763894.596766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274285568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763894.596800:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763894.596850:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763894/real 1787763894] req@ffff9ac4523bcf00 x1874603274285440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763894.596867:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274285440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763894.596874:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763894.596877:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763894.596881:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763894.596906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285568:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763894.596939:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763894.596956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763894/real 1787763894] req@ffff9ac4523bc780 x1874603274285568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763894.596964:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274285568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763894.596970:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763894.596973:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763894.596976:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285568:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763895.172686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763895.620647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763896.196642:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763896.644601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763897.220701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763897.668649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763898.244711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763898.692669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763899.268709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763899.716653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763899.716655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763899.716682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763899.716688:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763899.716698:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2944 00000100:00080000:1.0:1787763899.716710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763899.716717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763899.716755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274285696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763899.716780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763899.716782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763899.716784:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763899.716787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2944 00000100:00080000:1.0:1787763899.716789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763899.716792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763899.716801:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763899.716809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274285824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763899.716857:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285696:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763899.716945:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763899.716988:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763899/real 1787763899] req@ffff9ac447843840 x1874603274285696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763899.717002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274285696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763899.717011:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763899.717015:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763899.717020:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285696:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763899.717027:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285824:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763899.717065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763899.717079:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763899/real 1787763899] req@ffff9ac447841a40 x1874603274285824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763899.717090:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274285824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763899.717096:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763899.717099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763899.717101:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274285824:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763900.292701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763900.740684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763901.316737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763901.764661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763902.340736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763902.788663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763903.364686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763903.812631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763904.388777:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763904.836664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763904.836672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763904.836675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763904.836686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763904.836698:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2949 00000100:00080000:1.0:1787763904.836709:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763904.836718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763904.836778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274285952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763904.836805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763904.836808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763904.836809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763904.836813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2949 00000100:00080000:1.0:1787763904.836816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763904.836818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763904.836828:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763904.836850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274286080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763904.836899:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285952:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763904.836988:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763904.837042:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763904/real 1787763904] req@ffff9ac4478412c0 x1874603274285952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763904.837063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274285952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763904.837072:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763904.837075:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763904.837078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274285952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763904.837084:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286080:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763904.837104:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763904.837112:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763904/real 1787763904] req@ffff9ac447841e00 x1874603274286080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763904.837140:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274286080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763904.837146:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763904.837149:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763904.837151:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286080:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763905.412670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763905.860767:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763906.436679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763906.884660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763907.460705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763907.908660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763908.484740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763908.932656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763909.508678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763909.956623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763909.956632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763909.956636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763909.956639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763909.956646:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2954 00000100:00080000:1.0:1787763909.956659:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763909.956666:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763909.956703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274286208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763909.956719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763909.956723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763909.956725:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763909.956742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2954 00000100:00080000:1.0:1787763909.956745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763909.956748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763909.956757:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286208:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787763909.956758:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763909.956776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274286336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763909.956868:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763909.956919:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763909/real 1787763909] req@ffff9ac447843480 x1874603274286208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763909.956940:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274286208/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763909.956953:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763909.956957:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763909.956962:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286208:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763909.956990:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286336:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763909.957012:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763909.957024:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763909/real 1787763909] req@ffff9ac447841e00 x1874603274286336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763909.957038:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274286336/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763909.957074:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763909.957079:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763909.957083:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286336:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763910.532722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763910.980671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763911.556708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763912.004618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763912.580710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763913.028672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763913.604683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763914.052640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763914.628696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763915.076607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763915.076659:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763915.076669:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763915.076674:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763915.076684:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2959 00000100:00080000:3.0:1787763915.076696:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763915.076703:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763915.076764:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274286464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763915.076809:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763915.076812:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763915.076814:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763915.076817:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2959 00000100:00080000:3.0:1787763915.076821:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763915.076849:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763915.076860:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763915.076862:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763915.076877:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274286592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763915.076930:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763915.076965:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763915/real 1787763915] req@ffff9ac451c530c0 x1874603274286464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763915.076979:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274286464/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763915.076987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763915.076990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763915.076993:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286464:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763915.077015:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286592:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763915.077030:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763915.077038:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763915/real 1787763915] req@ffff9ac451c52d00 x1874603274286592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763915.077045:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274286592/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763915.077051:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763915.077054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763915.077056:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286592:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763915.652670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763916.100603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763916.676749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763917.124624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763917.700631:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763918.148574:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763918.724665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763919.172647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763919.748692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763920.196646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763920.196647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763920.196657:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763920.196662:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763920.196672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2964 00000100:00080000:3.0:1787763920.196684:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763920.196691:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763920.196727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274286720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763920.196772:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763920.196776:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763920.196778:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763920.196782:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2964 00000100:00080000:3.0:1787763920.196786:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763920.196792:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763920.196802:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763920.196820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763920.196820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274286848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763920.196878:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763920.196912:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763920/real 1787763920] req@ffff9ac451c50b40 x1874603274286720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763920.196930:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274286720/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763920.196941:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763920.196945:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763920.196950:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286720:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763920.196971:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286848:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763920.196990:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763920.196999:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763920/real 1787763920] req@ffff9ac451c52940 x1874603274286848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763920.197007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274286848/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763920.197013:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763920.197016:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763920.197018:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274286848:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763920.772685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763921.220630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763921.796747:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763922.244648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763922.820706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763923.268593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763923.844746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763924.292663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763924.868692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763925.316607:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763925.316614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763925.316616:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763925.316622:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763925.316631:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2970 00000100:00080000:3.0:1787763925.316640:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763925.316646:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763925.316676:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274286976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763925.316711:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763925.316714:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763925.316716:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763925.316719:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2970 00000100:00080000:3.0:1787763925.316722:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763925.316725:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763925.316733:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763925.316744:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274287104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763925.316755:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286976:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763925.316809:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763925.316842:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763925/real 1787763925] req@ffff9ac451c50780 x1874603274286976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763925.316869:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274286976/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763925.316880:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763925.316885:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763925.316889:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274286976:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763925.316912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287104:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763925.316934:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763925.316945:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763925/real 1787763925] req@ffff9ac451c503c0 x1874603274287104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763925.316953:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274287104/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763925.316960:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763925.316962:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763925.316964:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287104:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763925.892787:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763926.340652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763926.916714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763927.364631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763927.940756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763928.388643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763928.964688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763929.412610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763929.988717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763930.436682:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763930.436691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763930.436694:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763930.436715:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763930.436729:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2975 00000100:00080000:3.0:1787763930.436740:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763930.436748:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763930.436781:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274287232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763930.436815:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763930.436818:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763930.436820:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763930.436825:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2975 00000100:00080000:3.0:1787763930.436828:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763930.436832:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763930.436842:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763930.436854:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274287360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763930.436876:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274287232:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763930.436966:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287360:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787763931.012716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763931.460633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763932.036729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763932.484664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763933.060687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763933.508636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787763933.508838:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787763933.508900:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787763933.508923:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787763933.508966:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787763933.509009:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763930/real 1787763933] req@ffff9ac451c50f00 x1874603274287360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763933.509039:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274287360/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763933.509078:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763933.509084:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763933.509090:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287360:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787763933.509099:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763930/real 1787763933] req@ffff9ac451c52580 x1874603274287232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763933.509114:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274287232/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763933.509123:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763933.509126:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763933.509130:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274287232:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787763934.084720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763934.532643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787763935.108667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763935.556647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763935.556714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763935.556722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763935.556728:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763935.556741:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2980 00000100:00080000:3.0:1787763935.556754:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763935.556763:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763935.556818:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274287488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763935.556878:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763935.556883:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763935.556885:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763935.556890:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2980 00000100:00080000:3.0:1787763935.556894:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763935.556899:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763935.556919:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763935.556939:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274287616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763935.556941:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274287488:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763935.557029:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763935.557080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763935/real 1787763935] req@ffff9ac451c52580 x1874603274287488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763935.557097:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274287488/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763935.557109:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763935.557114:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763935.557129:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274287488:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763935.557164:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287616:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763935.557204:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763935.557220:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763935/real 1787763935] req@ffff9ac451c50f00 x1874603274287616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763935.557236:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274287616/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763935.557245:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763935.557248:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763935.557251:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287616:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763936.132687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763936.580651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763937.156637:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763937.604654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763938.180687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763938.628627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763939.204660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763939.652641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763940.228658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763940.676641:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763940.676643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763940.676651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763940.676654:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763940.676662:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2985 00000100:00080000:3.0:1787763940.676673:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763940.676679:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763940.676711:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274287744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763940.676732:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763940.676735:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763940.676749:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763940.676752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2985 00000100:00080000:3.0:1787763940.676756:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763940.676760:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763940.676769:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763940.676780:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274287872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763940.676816:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274287744:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763940.676904:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763940.676958:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763940/real 1787763940] req@ffff9ac451c503c0 x1874603274287744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763940.676974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274287744/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763940.676984:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763940.676988:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763940.676991:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274287744:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763940.677014:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287872:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763940.677031:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763940.677042:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763940/real 1787763940] req@ffff9ac451c50780 x1874603274287872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763940.677071:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274287872/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763940.677077:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763940.677081:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763940.677084:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274287872:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763941.252655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763941.700665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763942.276745:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763942.724638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763943.300740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763943.748632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763944.324718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763944.772620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763945.348698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787763945.796652:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763945.796663:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787763945.796664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763945.796668:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763945.796685:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2990 00000100:00080000:3.0:1787763945.796704:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763945.796712:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763945.796756:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274288000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763945.796790:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763945.796793:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763945.796794:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763945.796798:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2990 00000100:00080000:3.0:1787763945.796801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763945.796824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763945.796835:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763945.796845:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288000:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763945.796848:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274288128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763945.796922:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763945.796964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763945/real 1787763945] req@ffff9ac451c52940 x1874603274288000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763945.796979:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274288000/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763945.796989:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763945.796992:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763945.796995:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288000:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763945.797017:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288128:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763945.797032:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763945.797040:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763945/real 1787763945] req@ffff9ac451c50b40 x1874603274288128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763945.797048:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274288128/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763945.797065:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763945.797068:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763945.797070:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288128:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763946.372687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763946.820652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763947.396782:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763947.844642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763948.420699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763948.868607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763949.444656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763949.892655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763950.468688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763950.916598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763950.916635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763950.916641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763950.916644:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763950.916652:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 2995 00000100:00080000:1.0:1787763950.916677:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763950.916683:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763950.916711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274288256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763950.916733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763950.916736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763950.916737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763950.916744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 2995 00000100:00080000:1.0:1787763950.916747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763950.916749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763950.916757:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763950.916766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274288384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763950.916776:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288256:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763950.916829:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763950.916861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763950/real 1787763950] req@ffff9ac4478412c0 x1874603274288256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763950.916874:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274288256/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763950.916882:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763950.916885:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763950.916887:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288256:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763950.916904:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288384:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763950.916920:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763950.916928:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763950/real 1787763950] req@ffff9ac447841a40 x1874603274288384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763950.916936:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274288384/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763950.916942:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763950.916943:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763950.916946:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288384:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763951.492670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763951.940646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763952.516634:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763952.964612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763953.540688:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763953.988644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763954.564662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763955.012618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763955.588723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763956.036660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787763956.036698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763956.036703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787763956.036707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763956.036717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3000 00000100:00080000:2.0:1787763956.036729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763956.036737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763956.036780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274288512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787763956.036805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787763956.036808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787763956.036810:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787763956.036813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3000 00000100:00080000:2.0:1787763956.036815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787763956.036818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787763956.036827:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787763956.036837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274288640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763956.036866:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288512:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763956.036952:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763956.037003:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763956/real 1787763956] req@ffff9ac4523bd680 x1874603274288512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763956.037022:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274288512/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763956.037035:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763956.037039:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763956.037044:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288512:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763956.037052:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288640:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763956.037078:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763956.037092:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763956/real 1787763956] req@ffff9ac4523bc000 x1874603274288640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763956.037104:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274288640/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763956.037113:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763956.037117:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763956.037121:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288640:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763956.612710:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763957.060653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763957.636664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763958.084633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763958.660725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763959.108657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763959.684743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763960.132644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763960.708658:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763961.156640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787763961.156640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763961.156650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763961.156655:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763961.156667:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3005 00000100:00080000:1.0:1787763961.156679:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763961.156687:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763961.156727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274288768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763961.156751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763961.156754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763961.156762:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763961.156766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3005 00000100:00080000:1.0:1787763961.156769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763961.156772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763961.156782:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763961.156791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274288896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763961.156831:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288768:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763961.156903:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763961.156944:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763961/real 1787763961] req@ffff9ac447843840 x1874603274288768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763961.156966:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274288768/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763961.156975:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763961.156978:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763961.156981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274288768:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763961.156986:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288896:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763961.157003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763961.157011:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763961/real 1787763961] req@ffff9ac447842580 x1874603274288896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763961.157019:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274288896/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763961.157025:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763961.157027:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763961.157029:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274288896:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763961.732736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763962.180701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763962.756708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763963.204666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763963.780670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763964.228623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763964.804679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763965.252670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763965.828681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763966.276608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763966.276620:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763966.276628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763966.276633:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763966.276643:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3011 00000100:00080000:3.0:1787763966.276653:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763966.276677:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763966.276706:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274289024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763966.276729:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763966.276732:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763966.276733:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763966.276736:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3011 00000100:00080000:3.0:1787763966.276740:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763966.276743:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763966.276751:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763966.276758:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289024:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787763966.276763:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274289152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787763966.276814:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763966.276847:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763966/real 1787763966] req@ffff9ac451c52d00 x1874603274289024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763966.276867:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274289024/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763966.276875:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763966.276879:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763966.276881:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289024:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763966.276906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289152:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763966.276931:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763966.276944:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763966/real 1787763966] req@ffff9ac451c530c0 x1874603274289152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763966.276957:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274289152/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763966.276966:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763966.276969:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763966.276972:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289152:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763966.852701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763967.300656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763967.876659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763968.324644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763968.900714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763969.348728:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763969.924657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763970.372667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763970.948699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763971.396657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763971.396657:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763971.396669:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763971.396673:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763971.396682:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3016 00000100:00080000:3.0:1787763971.396693:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763971.396701:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763971.396748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274289280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763971.396806:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763971.396810:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763971.396812:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763971.396816:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3016 00000100:00080000:3.0:1787763971.396820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763971.396825:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763971.396835:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763971.396849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274289408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787763971.396857:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289280:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787763971.396931:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763971.396969:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763971/real 1787763971] req@ffff9ac451c530c0 x1874603274289280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763971.396986:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274289280/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763971.397004:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763971.397008:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763971.397012:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289280:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787763971.397056:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289408:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787763971.397082:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787763971.397111:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763971/real 1787763971] req@ffff9ac451c52d00 x1874603274289408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787763971.397123:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274289408/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787763971.397130:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787763971.397133:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787763971.397136:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289408:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763971.972702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763972.420596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763972.996695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763973.444651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763974.020650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763974.468603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763975.044707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763975.492682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763976.068651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787763976.516667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763976.516677:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763976.516682:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787763976.516688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763976.516695:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3021 00000100:00080000:0.0:1787763976.516711:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763976.516720:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763976.516755:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274289536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763976.516788:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763976.516808:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763976.516812:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763976.516818:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3021 00000100:00080000:0.0:1787763976.516823:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763976.516828:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787763976.516836:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289536:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787763976.516841:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763976.516865:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274289664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787763976.516932:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763976.516986:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763976/real 1787763976] req@ffff9ac451f84780 x1874603274289536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763976.517009:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274289536/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763976.517023:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763976.517029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763976.517034:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289536:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763976.517062:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289664:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763976.517088:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763976.517114:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763976/real 1787763976] req@ffff9ac451f861c0 x1874603274289664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763976.517131:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274289664/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763976.517138:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763976.517140:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763976.517155:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289664:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763977.092719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763977.540668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763978.116701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763978.564631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763979.140649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763979.588659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763980.164761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763980.612676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763981.188740:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763981.636605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787763981.636606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763981.636618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787763981.636622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763981.636632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3026 00000100:00080000:0.0:1787763981.636642:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763981.636648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787763981.636679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274289792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787763981.636702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787763981.636705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787763981.636707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787763981.636709:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3026 00000100:00080000:0.0:1787763981.636712:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787763981.636715:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787763981.636723:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787763981.636732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274289920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763981.636738:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289792:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763981.636820:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763981.636873:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763981/real 1787763981] req@ffff9ac451f870c0 x1874603274289792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763981.636889:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274289792/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763981.636899:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763981.636902:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763981.636906:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274289792:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763981.636929:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289920:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763981.636946:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763981.636963:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763981/real 1787763981] req@ffff9ac451f84000 x1874603274289920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763981.636972:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274289920/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763981.636978:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763981.636981:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763981.636995:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274289920:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763982.212750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763982.660634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763983.236686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763983.684631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763984.260785:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763984.708649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763985.284722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763985.732640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763986.308733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763986.756643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763986.756672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763986.756682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763986.756686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763986.756694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3031 00000100:00080000:1.0:1787763986.756704:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763986.756710:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763986.756742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274290048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763986.756768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763986.756771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763986.756772:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763986.756775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3031 00000100:00080000:1.0:1787763986.756779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763986.756784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763986.756793:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763986.756804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274290176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763986.756813:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290048:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763986.756908:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763986.756966:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763986/real 1787763986] req@ffff9ac447840000 x1874603274290048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763986.756983:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274290048/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763986.756994:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763986.756998:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763986.757001:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290048:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763986.757024:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290176:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763986.757041:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763986.757050:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763986/real 1787763986] req@ffff9ac447840f00 x1874603274290176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763986.757058:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274290176/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763986.757064:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763986.757067:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763986.757081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290176:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763987.332718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763987.780611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763988.356736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763988.804658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763989.380703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763989.828600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763990.404716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763990.852667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763991.428664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763991.876605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787763991.876626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763991.876635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787763991.876638:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763991.876647:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3036 00000100:00080000:3.0:1787763991.876656:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763991.876662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787763991.876699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274290304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787763991.876719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787763991.876722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787763991.876723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787763991.876726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3036 00000100:00080000:3.0:1787763991.876729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787763991.876731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787763991.876739:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787763991.876750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274290432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763991.876765:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290304:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787763991.876843:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763991.876895:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763991/real 1787763991] req@ffff9ac451c50b40 x1874603274290304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763991.876910:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274290304/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763991.876919:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763991.876923:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763991.876926:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290304:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763991.876951:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290432:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787763991.876969:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787763991.876978:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763991/real 1787763991] req@ffff9ac451c52940 x1874603274290432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787763991.876986:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274290432/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787763991.876993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787763991.876995:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787763991.877015:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290432:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763992.452700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787763992.900689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763993.476645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763993.924608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763994.500701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763994.948638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763995.524657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763995.972661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763996.548695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787763996.996666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787763996.996667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787763996.996675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787763996.996679:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763996.996687:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3041 00000100:00080000:1.0:1787763996.996696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763996.996702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787763996.996734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274290560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787763996.996758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787763996.996763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787763996.996765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787763996.996769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3041 00000100:00080000:1.0:1787763996.996774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787763996.996779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787763996.996790:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787763996.996806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274290688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787763996.996824:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290560:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787763996.996979:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290688:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787763997.572680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763998.020664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763998.596692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787763999.044666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787763999.620782:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764000.068644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787764000.068783:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787764000.068836:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787764000.068851:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787764000.068883:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787764000.068922:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763996/real 1787764000] req@ffff9ac447841680 x1874603274290688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764000.068946:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274290688/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764000.068960:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764000.068966:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764000.068971:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290688:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787764000.068980:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763996/real 1787764000] req@ffff9ac447840780 x1874603274290560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764000.068993:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274290560/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764000.069002:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764000.069006:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764000.069010:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290560:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787764000.644712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764001.092739:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764001.668680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764002.116652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764002.116661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764002.116679:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764002.116688:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3046 00000100:00080000:1.0:1787764002.116698:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764002.116704:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764002.116729:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787764002.116740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274290816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764002.116771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764002.116775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764002.116776:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764002.116780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3046 00000100:00080000:1.0:1787764002.116783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764002.116786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764002.116794:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764002.116804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274290944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764002.116852:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290816:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764002.116948:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764002.117013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764002/real 1787764002] req@ffff9ac4478403c0 x1874603274290816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764002.117053:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274290816/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764002.117064:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764002.117069:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764002.117074:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274290816:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764002.117083:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290944:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764002.117111:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764002.117147:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764002/real 1787764002] req@ffff9ac4478430c0 x1874603274290944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764002.117160:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274290944/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764002.117186:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764002.117190:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764002.117194:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274290944:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764002.692665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764003.140785:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764003.716661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764004.164669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764004.740677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764005.188622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764005.764689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764006.212659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764006.788673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764007.236613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764007.236643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764007.236653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764007.236658:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764007.236670:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3051 00000100:00080000:3.0:1787764007.236681:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764007.236689:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764007.236724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274291072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764007.236752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764007.236756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764007.236759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764007.236763:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3051 00000100:00080000:3.0:1787764007.236767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764007.236772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764007.236783:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764007.236797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274291200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764007.236811:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291072:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764007.236897:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764007.236947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764007/real 1787764007] req@ffff9ac451c50780 x1874603274291072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764007.236967:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274291072/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764007.237007:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764007.237013:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764007.237019:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291072:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764007.237048:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291200:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764007.237075:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764007.237089:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764007/real 1787764007] req@ffff9ac451c503c0 x1874603274291200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764007.237103:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274291200/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764007.237113:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764007.237117:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764007.237121:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291200:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764007.812677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764008.260642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764008.836653:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764009.284626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764009.860679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764010.308646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764010.884703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764011.332630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764011.908778:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764012.356670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764012.356681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764012.356686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787764012.356690:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764012.356699:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3057 00000100:00080000:1.0:1787764012.356718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764012.356726:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764012.356769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274291328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764012.356793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764012.356814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764012.356816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764012.356820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3057 00000100:00080000:1.0:1787764012.356824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764012.356827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764012.356837:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764012.356848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274291456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764012.356855:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291328:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764012.356931:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764012.356974:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764012/real 1787764012] req@ffff9ac4478421c0 x1874603274291328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764012.356990:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274291328/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764012.357000:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764012.357003:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764012.357007:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291328:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764012.357041:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291456:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764012.357062:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764012.357071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764012/real 1787764012] req@ffff9ac447842d00 x1874603274291456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764012.357079:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274291456/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764012.357085:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764012.357088:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764012.357110:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291456:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764012.932690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764013.380600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764013.956736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764014.404623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764014.980680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764015.428646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764016.004724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764016.452689:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764017.028725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764017.476686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764017.476690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764017.476700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764017.476705:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764017.476717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3062 00000100:00080000:1.0:1787764017.476728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764017.476736:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764017.476783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274291584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764017.476807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764017.476810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764017.476812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764017.476815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3062 00000100:00080000:1.0:1787764017.476818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764017.476821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764017.476830:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764017.476840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274291712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764017.476898:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291584:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764017.477017:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764017.477086:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764017/real 1787764017] req@ffff9ac447840b40 x1874603274291584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764017.477111:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274291584/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764017.477125:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764017.477131:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764017.477136:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291584:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764017.477146:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291712:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764017.477178:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764017.477194:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764017/real 1787764017] req@ffff9ac447842940 x1874603274291712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764017.477221:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274291712/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764017.477232:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764017.477237:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764017.477241:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291712:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764018.052727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764018.500656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764019.076704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764019.524675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764020.100697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764020.548656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764021.124760:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764021.572630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764022.148677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764022.596601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764022.596639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764022.596646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764022.596651:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764022.596660:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3067 00000100:00080000:3.0:1787764022.596672:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764022.596679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764022.596722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274291840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764022.596754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764022.596758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764022.596761:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764022.596766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3067 00000100:00080000:3.0:1787764022.596770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764022.596776:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764022.596799:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291840:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787764022.596807:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764022.596831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274291968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764022.596878:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764022.596926:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764022/real 1787764022] req@ffff9ac451c50f00 x1874603274291840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764022.596941:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274291840/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764022.596950:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764022.596954:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764022.596957:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274291840:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764022.596980:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291968:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764022.596998:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764022.597007:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764022/real 1787764022] req@ffff9ac451c52580 x1874603274291968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764022.597016:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274291968/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764022.597045:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764022.597048:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764022.597051:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274291968:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764023.172725:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764023.620601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764024.196790:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764024.644628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764025.220704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764025.668638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764026.244694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764026.692620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764027.268672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764027.716621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764027.716621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764027.716631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764027.716636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764027.716645:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3072 00000100:00080000:3.0:1787764027.716657:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764027.716665:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764027.716705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274292096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764027.716733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764027.716738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764027.716740:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764027.716745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3072 00000100:00080000:3.0:1787764027.716749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764027.716755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764027.716765:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764027.716778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274292224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764027.716783:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292096:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764027.716841:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764027.716884:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764027/real 1787764027] req@ffff9ac451c52580 x1874603274292096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764027.716897:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274292096/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764027.716904:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764027.716907:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764027.716909:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292096:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764027.716928:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292224:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764027.716942:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764027.716950:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764027/real 1787764027] req@ffff9ac451c50f00 x1874603274292224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764027.716957:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274292224/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764027.716963:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764027.716965:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764027.716967:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292224:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764028.292669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764028.740608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764029.316684:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764029.764617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764030.340678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764030.788597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764031.364720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764031.812616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764032.388649:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764032.836619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764032.836633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764032.836639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764032.836644:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764032.836656:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3077 00000100:00080000:1.0:1787764032.836670:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764032.836680:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764032.836731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274292352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764032.836755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764032.836758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764032.836760:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764032.836763:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3077 00000100:00080000:1.0:1787764032.836766:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764032.836770:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764032.836779:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764032.836793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274292480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764032.836812:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292352:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764032.836886:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764032.836927:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764032/real 1787764032] req@ffff9ac447843c00 x1874603274292352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764032.836942:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274292352/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764032.836966:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764032.836971:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764032.836975:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292352:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764032.837004:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292480:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764032.837029:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764032.837042:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764032/real 1787764032] req@ffff9ac447842940 x1874603274292480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764032.837056:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274292480/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764032.837066:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764032.837069:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764032.837073:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292480:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764033.412757:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764033.860661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764034.436734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764034.884684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764035.460729:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764035.908624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764036.484743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764036.932704:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764037.508791:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764037.956657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764037.956660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764037.956669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764037.956674:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764037.956685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3082 00000100:00080000:1.0:1787764037.956698:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764037.956705:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764037.956786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274292608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764037.956821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764037.956824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764037.956829:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764037.956832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3082 00000100:00080000:1.0:1787764037.956835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764037.956839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764037.956849:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764037.956859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274292736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764037.956875:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292608:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764037.956977:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764037.957021:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764037/real 1787764037] req@ffff9ac447840b40 x1874603274292608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764037.957042:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274292608/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764037.957051:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764037.957054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764037.957065:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292608:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764037.957088:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292736:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764037.957108:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764037.957118:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764037/real 1787764037] req@ffff9ac447842d00 x1874603274292736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764037.957126:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274292736/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764037.957132:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764037.957134:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764037.957136:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292736:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764038.532748:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764038.980658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764039.556752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764040.004651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764040.580669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764041.028690:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764041.604759:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764042.052651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764042.628749:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764043.076594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764043.076641:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764043.076649:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764043.076653:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764043.076662:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3087 00000100:00080000:3.0:1787764043.076672:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764043.076678:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764043.076709:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274292864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764043.076739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764043.076742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764043.076744:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764043.076746:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3087 00000100:00080000:3.0:1787764043.076749:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764043.076768:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764043.076777:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764043.076791:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787764043.076792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274292992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764043.076851:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764043.076895:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764043/real 1787764043] req@ffff9ac451c503c0 x1874603274292864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764043.076909:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274292864/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764043.076928:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764043.076931:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764043.076934:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274292864:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764043.076955:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292992:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764043.076975:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764043.076984:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764043/real 1787764043] req@ffff9ac451c50780 x1874603274292992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764043.076992:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274292992/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764043.076998:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764043.077001:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764043.077003:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274292992:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764043.652719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764044.100620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764044.676746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764045.124658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764045.700705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764046.148589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764046.724797:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764047.172658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764047.748671:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764048.196611:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764048.196619:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764048.196622:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764048.196630:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3092 00000100:00080000:1.0:1787764048.196640:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764048.196646:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764048.196658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787764048.196681:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274293120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764048.196704:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764048.196722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764048.196724:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764048.196728:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3092 00000100:00080000:1.0:1787764048.196731:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764048.196735:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764048.196743:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764048.196749:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764048.196757:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274293248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764048.196822:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764048.196873:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764048/real 1787764048] req@ffff9ac4478421c0 x1874603274293120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764048.196889:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274293120/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764048.196898:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764048.196902:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764048.196905:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293120:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764048.196936:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274293248:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764048.196956:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764048.196965:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764048/real 1787764048] req@ffff9ac4478430c0 x1874603274293248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764048.196974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274293248/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764048.196980:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764048.196982:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764048.196985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274293248:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764048.772696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764049.220680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764049.796681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764050.244608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764050.820664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764051.268659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764051.844729:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764052.292630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764052.868728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764053.316652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764053.316664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764053.316673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764053.316677:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764053.316688:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3098 00000100:00080000:2.0:1787764053.316699:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764053.316707:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764053.316746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274293376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764053.316776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764053.316780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764053.316782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764053.316786:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3098 00000100:00080000:2.0:1787764053.316790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764053.316831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764053.316832:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293376:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787764053.316845:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764053.316865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274293504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764053.316902:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764053.316946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764053/real 1787764053] req@ffff9ac4523be940 x1874603274293376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764053.316974:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274293376/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764053.316986:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764053.316991:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764053.316995:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293376:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764053.317022:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274293504:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764053.317048:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764053.317063:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764053/real 1787764053] req@ffff9ac4523bc3c0 x1874603274293504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764053.317077:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274293504/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764053.317085:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764053.317089:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764053.317093:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274293504:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764053.892639:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764054.340630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764054.916703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764055.364639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764055.940722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764056.388664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764056.964717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764057.412615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764057.988712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764058.436636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764058.436677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764058.436683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764058.436686:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764058.436694:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3103 00000100:00080000:3.0:1787764058.436704:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764058.436709:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764058.436744:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274293632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764058.436766:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764058.436769:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764058.436771:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764058.436774:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3103 00000100:00080000:3.0:1787764058.436788:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764058.436791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764058.436806:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293632:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787764058.436820:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764058.436835:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274293760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764058.436869:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764058.436912:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764058/real 1787764058] req@ffff9ac451c52940 x1874603274293632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764058.436938:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274293632/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764058.436946:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764058.436949:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764058.436952:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293632:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764058.436983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274293760:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764058.437008:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764058.437022:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764058/real 1787764058] req@ffff9ac451c50b40 x1874603274293760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764058.437033:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274293760/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764058.437039:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764058.437042:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764058.437044:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274293760:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764059.012700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764059.460666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764060.036790:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764060.484627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764061.060670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764061.508662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764062.084651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764062.532651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764063.108699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764063.556659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764063.556662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764063.556671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764063.556676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764063.556685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3108 00000100:00080000:2.0:1787764063.556697:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764063.556705:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764063.556751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274293888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764063.556780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764063.556783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764063.556785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764063.556790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3108 00000100:00080000:2.0:1787764063.556794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764063.556799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764063.556810:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764063.556814:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293888:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764063.556828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc3c0 x1874603274294016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764063.556917:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294016:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764064.132680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764064.580601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764065.156628:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764065.604606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764066.180641:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764066.628630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:3.0:1787764066.628774:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:3.0:1787764066.628806:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787764066.628820:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:3.0:1787764066.628850:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787764066.628879:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764063/real 1787764066] req@ffff9ac4523bc3c0 x1874603274294016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764066.628899:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc3c0 x1874603274294016/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764066.628908:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764066.628912:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764066.628915:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294016:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787764066.628921:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764063/real 1787764066] req@ffff9ac4523bda40 x1874603274293888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764066.628929:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274293888/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764066.628934:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764066.628937:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764066.628939:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274293888:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787764067.204693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764067.652633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764068.228696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764068.676648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764068.676680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764068.676692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764068.676698:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764068.676726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3113 00000100:00080000:1.0:1787764068.676740:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764068.676750:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764068.676795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274294144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764068.676855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764068.676860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764068.676862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764068.676868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3113 00000100:00080000:1.0:1787764068.676872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764068.676879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764068.676892:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764068.676914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274294272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764068.676916:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294144:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764068.677012:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764068.677064:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764068/real 1787764068] req@ffff9ac4478403c0 x1874603274294144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764068.677105:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274294144/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764068.677115:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764068.677121:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764068.677126:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294144:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764068.677152:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294272:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764068.677176:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764068.677190:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764068/real 1787764068] req@ffff9ac447840780 x1874603274294272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764068.677202:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274294272/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764068.677210:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764068.677213:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764068.677217:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294272:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764069.252691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764069.700606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764070.276651:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764070.724622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764071.300647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764071.748619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764072.324632:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764072.772610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764073.348640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764073.796650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764073.796697:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764073.796702:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764073.796706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764073.796713:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3118 00000100:00080000:3.0:1787764073.796723:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764073.796728:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764073.796760:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274294400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764073.796783:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764073.796786:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764073.796788:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764073.796805:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3118 00000100:00080000:3.0:1787764073.796809:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764073.796812:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764073.796820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294400:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787764073.796821:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764073.796834:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274294528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764073.796879:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764073.796940:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764073/real 1787764073] req@ffff9ac451c52d00 x1874603274294400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764073.796965:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274294400/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764073.796974:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764073.796977:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764073.796981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294400:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764073.797005:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294528:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764073.797025:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764073.797036:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764073/real 1787764073] req@ffff9ac451c530c0 x1874603274294528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764073.797046:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274294528/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764073.797052:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764073.797055:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764073.797057:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294528:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764074.372710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764074.820651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764075.396734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764075.844670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764076.420767:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764076.868662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764077.444762:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764077.892701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764078.468705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764078.916696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787764078.916702:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764078.916704:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764078.916709:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764078.916722:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3123 00000100:00080000:3.0:1787764078.916734:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764078.916740:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764078.916774:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274294656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764078.916798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764078.916801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764078.916803:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764078.916805:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3123 00000100:00080000:3.0:1787764078.916807:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764078.916810:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764078.916831:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764078.916841:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274294784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764078.916874:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294656:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764078.917006:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764078.917084:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764078/real 1787764078] req@ffff9ac451c530c0 x1874603274294656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764078.917170:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274294656/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764078.917208:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764078.917216:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764078.917224:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294656:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764078.917285:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294784:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764078.917312:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764078.917344:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764078/real 1787764078] req@ffff9ac451c52d00 x1874603274294784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764078.917358:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274294784/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764078.917368:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764078.917371:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764078.917375:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274294784:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764079.492733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764079.940706:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764080.516718:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764080.964593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764081.540709:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764081.988710:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764082.564687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764083.012661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764083.588661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764084.036669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764084.036707:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764084.036716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764084.036720:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764084.036728:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3128 00000100:00080000:1.0:1787764084.036739:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764084.036746:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764084.036777:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274294912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764084.036796:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764084.036800:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764084.036801:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764084.036804:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3128 00000100:00080000:1.0:1787764084.036820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764084.036824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764084.036828:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294912:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787764084.036835:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764084.036852:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274295040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764084.036890:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764084.036922:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764084/real 1787764084] req@ffff9ac447841680 x1874603274294912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764084.036942:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274294912/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764084.036950:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764084.036952:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764084.036956:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274294912:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764084.036974:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295040:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764084.036990:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764084.036999:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764084/real 1787764084] req@ffff9ac447840f00 x1874603274295040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764084.037007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274295040/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764084.037014:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764084.037016:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764084.037019:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295040:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764084.612662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764085.060602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764085.636679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764086.084654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764086.660697:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764087.108639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764087.684718:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764088.132658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764088.708712:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764089.156643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764089.156643:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764089.156656:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764089.156661:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764089.156672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3133 00000100:00080000:1.0:1787764089.156683:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764089.156691:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764089.156736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274295168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764089.156769:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764089.156774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764089.156776:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764089.156782:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3133 00000100:00080000:1.0:1787764089.156787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764089.156793:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764089.156805:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764089.156806:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295168:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764089.156826:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274295296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764089.156880:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764089.156922:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764089/real 1787764089] req@ffff9ac447840000 x1874603274295168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764089.156957:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274295168/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764089.156970:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764089.156975:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764089.156979:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295168:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764089.157007:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295296:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764089.157032:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764089.157042:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764089/real 1787764089] req@ffff9ac447842580 x1874603274295296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764089.157051:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274295296/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764089.157057:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764089.157060:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764089.157062:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295296:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764089.732743:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764090.180615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764090.756673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764091.204687:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764091.780689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764092.228663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764092.804690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764093.252692:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764093.828694:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764094.276691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764094.276716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764094.276724:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764094.276728:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764094.276737:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3139 00000100:00080000:3.0:1787764094.276747:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764094.276753:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764094.276785:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274295424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764094.276807:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764094.276809:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764094.276825:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764094.276829:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3139 00000100:00080000:3.0:1787764094.276833:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764094.276838:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764094.276846:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764094.276849:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787764094.276860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274295552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764094.276938:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764094.276983:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764094/real 1787764094] req@ffff9ac451c50b40 x1874603274295424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764094.277013:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274295424/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764094.277023:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764094.277027:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764094.277031:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295424:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764094.277057:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295552:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764094.277078:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764094.277089:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764094/real 1787764094] req@ffff9ac451c52940 x1874603274295552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764094.277099:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274295552/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764094.277106:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764094.277109:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764094.277111:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295552:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764094.852682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764095.300650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764095.876682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764096.324602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764096.900670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764097.348684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764097.924677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764098.372595:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764098.948659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764099.396678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764099.396681:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764099.396695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764099.396702:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764099.396717:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3144 00000100:00080000:3.0:1787764099.396734:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764099.396748:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764099.396815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274295680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764099.396856:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764099.396860:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764099.396863:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764099.396869:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3144 00000100:00080000:3.0:1787764099.396875:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787764099.396907:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295680:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787764099.396913:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764099.396936:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764099.396960:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274295808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764099.397009:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764099.397061:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764099/real 1787764099] req@ffff9ac451c50780 x1874603274295680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764099.397097:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274295680/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764099.397111:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764099.397117:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764099.397122:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295680:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764099.397169:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295808:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764099.397197:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764099.397211:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764099/real 1787764099] req@ffff9ac451c503c0 x1874603274295808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764099.397226:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274295808/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764099.397236:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764099.397239:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764099.397243:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274295808:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764099.972775:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764100.420685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764100.996732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764101.444618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764102.020673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764102.468598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764103.044740:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764103.492661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764104.068663:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764104.516636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764104.516676:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764104.516687:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764104.516693:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764104.516705:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3149 00000100:00080000:3.0:1787764104.516718:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764104.516727:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764104.516770:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274295936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764104.516803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764104.516808:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764104.516810:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764104.516815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3149 00000100:00080000:3.0:1787764104.516818:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764104.516825:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764104.516838:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764104.516852:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274296064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764104.516864:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295936:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764104.516974:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764104.517044:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764104/real 1787764104] req@ffff9ac451c50f00 x1874603274295936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764104.517081:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274295936/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764104.517095:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764104.517101:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764104.517106:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274295936:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764104.517139:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296064:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764104.517178:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764104.517193:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764104/real 1787764104] req@ffff9ac451c52580 x1874603274296064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764104.517204:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274296064/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764104.517211:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764104.517224:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764104.517227:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296064:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764105.092801:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764105.540675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764106.116661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764106.564602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764107.140667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764107.588630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764108.164684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764108.612589:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764109.188749:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764109.636608:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764109.636620:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764109.636625:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764109.636635:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3154 00000100:00100000:0.0:1787764109.636638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764109.636647:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764109.636657:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764109.636697:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274296192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764109.636719:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764109.636722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764109.636723:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764109.636727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3154 00000100:00080000:3.0:1787764109.636729:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764109.636732:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764109.636740:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764109.636750:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274296320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764109.636759:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296192:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764109.636825:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764109.636869:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764109/real 1787764109] req@ffff9ac451c52580 x1874603274296192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764109.636904:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274296192/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764109.636916:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764109.636921:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764109.636926:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296192:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764109.636952:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296320:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764109.636978:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764109.636992:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764109/real 1787764109] req@ffff9ac451c50f00 x1874603274296320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764109.637007:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274296320/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764109.637017:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764109.637020:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764109.637024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296320:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764110.212654:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764110.660612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764111.236682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764111.684664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764112.260715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764112.708654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764113.284693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764113.732653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764114.308729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764114.756652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764114.756661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764114.756667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764114.756687:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764114.756697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3159 00000100:00080000:2.0:1787764114.756707:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764114.756715:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764114.756747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274296448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764114.756768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764114.756771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764114.756773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764114.756776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3159 00000100:00080000:2.0:1787764114.756778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764114.756781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764114.756789:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764114.756797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274296576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764114.756810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296448:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764114.756884:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764114.756933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764114/real 1787764114] req@ffff9ac4523be940 x1874603274296448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764114.756961:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274296448/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764114.756973:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764114.756986:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764114.756991:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296448:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764114.757015:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296576:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764114.757041:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764114.757054:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764114/real 1787764114] req@ffff9ac4523bc000 x1874603274296576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764114.757068:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274296576/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764114.757077:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764114.757081:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764114.757084:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296576:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764115.332691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764115.780617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764116.356687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764116.804669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764117.380726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764117.828627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764118.404723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764118.852665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764119.428638:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764119.876625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764119.876688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764119.876694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764119.876698:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764119.876708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3164 00000100:00080000:0.0:1787764119.876729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764119.876737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764119.876771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274296704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764119.876799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764119.876804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764119.876806:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764119.876810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3164 00000100:00080000:0.0:1787764119.876815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764119.876820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764119.876831:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764119.876847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274296832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764119.876873:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296704:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764119.876951:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764119.876995:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764119/real 1787764119] req@ffff9ac451f84b40 x1874603274296704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764119.877015:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274296704/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764119.877027:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764119.877031:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764119.877042:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296704:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764119.877048:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296832:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764119.877069:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764119.877079:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764119/real 1787764119] req@ffff9ac451f87c00 x1874603274296832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764119.877091:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274296832/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764119.877101:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764119.877105:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764119.877108:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274296832:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764120.452730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764120.900668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764121.476675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764121.924623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764122.500746:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764122.948645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764123.524692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764123.972654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764124.548750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764124.996682:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787764124.996685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764124.996711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764124.996716:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764124.996726:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3169 00000100:00080000:1.0:1787764124.996737:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764124.996744:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764124.996783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274296960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764124.996808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764124.996811:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764124.996813:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764124.996816:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3169 00000100:00080000:1.0:1787764124.996818:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764124.996821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764124.996830:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764124.996840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274297088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764124.996898:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296960:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764124.996973:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764124.997018:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764124/real 1787764124] req@ffff9ac447843840 x1874603274296960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764124.997039:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274296960/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764124.997048:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764124.997053:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764124.997057:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274296960:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764124.997066:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297088:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764124.997089:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764124.997121:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764124/real 1787764124] req@ffff9ac447841a40 x1874603274297088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764124.997135:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274297088/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764124.997145:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764124.997149:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764124.997153:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297088:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764125.572672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764126.020584:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764126.596713:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764127.044660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764127.620701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764128.068608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764128.644645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764129.092658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764129.668658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764130.116613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764130.116642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764130.116651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764130.116655:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764130.116665:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3174 00000100:00080000:0.0:1787764130.116678:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764130.116688:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764130.116759:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274297216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764130.116789:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764130.116794:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764130.116796:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764130.116802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3174 00000100:00080000:0.0:1787764130.116808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764130.116814:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764130.116827:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764130.116833:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297216:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764130.116847:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274297344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764130.116960:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297344:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764130.692703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764131.140596:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764131.716668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764132.164646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764132.740688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764133.188601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787764133.188914:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787764133.188969:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787764133.188991:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787764133.189045:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787764133.189134:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764130/real 1787764133] req@ffff9ac451f84b40 x1874603274297344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764133.189164:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274297344/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764133.189178:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764133.189183:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764133.189188:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297344:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787764133.189218:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764130/real 1787764133] req@ffff9ac451f87c00 x1874603274297216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764133.189234:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274297216/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764133.189242:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764133.189246:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764133.189249:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297216:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787764133.764771:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764134.212690:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764134.788640:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764135.236615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764135.236653:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764135.236658:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764135.236662:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764135.236669:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3179 00000100:00080000:1.0:1787764135.236690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764135.236696:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764135.236724:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274297472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764135.236744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764135.236747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764135.236748:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764135.236751:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3179 00000100:00080000:1.0:1787764135.236753:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764135.236756:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764135.236764:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764135.236773:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274297600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764135.236777:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297472:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764135.236846:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764135.236886:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764135/real 1787764135] req@ffff9ac4478412c0 x1874603274297472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764135.236905:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274297472/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764135.236915:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764135.236919:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764135.236923:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297472:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764135.236945:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297600:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764135.236970:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764135.236984:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764135/real 1787764135] req@ffff9ac447841e00 x1874603274297600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764135.236997:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274297600/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764135.237014:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764135.237018:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764135.237022:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297600:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764135.812696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764136.260666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764136.836635:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764137.284629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764137.860680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764138.308602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764138.884645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764139.332636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764139.908686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764140.356601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787764140.356606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764140.356616:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764140.356622:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764140.356635:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3185 00000100:00080000:1.0:1787764140.356648:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764140.356658:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764140.356702:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274297728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764140.356740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764140.356762:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764140.356764:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764140.356770:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3185 00000100:00080000:1.0:1787764140.356775:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764140.356781:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764140.356796:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764140.356819:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297728:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764140.356822:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274297856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764140.356896:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764140.356933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764140/real 1787764140] req@ffff9ac447843480 x1874603274297728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764140.356947:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274297728/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764140.356957:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764140.356960:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764140.356963:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297728:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764140.356982:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297856:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764140.356996:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764140.357004:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764140/real 1787764140] req@ffff9ac447841e00 x1874603274297856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764140.357012:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274297856/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764140.357018:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764140.357020:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764140.357022:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274297856:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764140.932645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764141.380631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764141.956789:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764142.404648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764142.980702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764143.428670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764144.004831:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764144.452693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764145.028680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764145.476654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764145.476655:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764145.476671:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764145.476676:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764145.476687:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3190 00000100:00080000:1.0:1787764145.476699:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764145.476708:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764145.476747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274297984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764145.476778:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764145.476782:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764145.476784:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764145.476789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3190 00000100:00080000:1.0:1787764145.476793:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764145.476797:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764145.476821:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764145.476837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274298112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764145.476845:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297984:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764145.476973:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764145.477029:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764145/real 1787764145] req@ffff9ac4478412c0 x1874603274297984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764145.477053:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274297984/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764145.477067:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764145.477072:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764145.477077:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274297984:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764145.477116:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298112:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764145.477144:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764145.477160:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764145/real 1787764145] req@ffff9ac447841a40 x1874603274298112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764145.477187:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274298112/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764145.477198:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764145.477201:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764145.477204:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298112:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764146.052772:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764146.500669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764147.076700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764147.524594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764148.100766:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764148.548649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764149.124693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764149.572678:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764150.148727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764150.596617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764150.596663:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764150.596672:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764150.596677:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764150.596712:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3195 00000100:00080000:2.0:1787764150.596725:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764150.596735:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764150.596780:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274298240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764150.596813:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764150.596817:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764150.596818:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764150.596823:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3195 00000100:00080000:2.0:1787764150.596827:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764150.596831:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764150.596847:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764150.596856:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274298240:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764150.596863:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274298368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764150.596928:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764150.596965:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764150/real 1787764150] req@ffff9ac4523bd680 x1874603274298240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764150.596982:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274298240/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764150.596993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764150.596997:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764150.597000:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274298240:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764150.597023:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298368:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764150.597041:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764150.597051:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764150/real 1787764150] req@ffff9ac4523bc780 x1874603274298368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764150.597063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274298368/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764150.597071:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764150.597074:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764150.597077:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298368:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764151.172655:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764151.620632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764152.196716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764152.644626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764153.220733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764153.668676:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764154.244761:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764154.692656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764155.268733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764155.716659:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787764155.716660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764155.716671:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764155.716676:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764155.716686:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3200 00000100:00080000:1.0:1787764155.716697:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764155.716705:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764155.716749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274298496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764155.716777:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764155.716780:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764155.716781:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764155.716786:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3200 00000100:00080000:1.0:1787764155.716792:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764155.716797:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764155.716808:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764155.716813:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274298496:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764155.716822:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274298624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764155.716901:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764155.716947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764155/real 1787764155] req@ffff9ac447843840 x1874603274298496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764155.716991:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274298496/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764155.717006:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764155.717012:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764155.717017:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274298496:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764155.717065:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298624:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764155.717102:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764155.717138:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764155/real 1787764155] req@ffff9ac447842580 x1874603274298624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764155.717169:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274298624/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764155.717181:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764155.717187:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764155.717191:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298624:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764156.292760:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764156.740663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764157.316768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764157.764626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764158.340710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764158.788639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764159.364755:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764159.812661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764160.388745:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764160.836633:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787764160.836641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764160.836644:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764160.836650:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764160.836664:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3205 00000100:00080000:1.0:1787764160.836676:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764160.836686:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764160.836735:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274298752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764160.836763:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764160.836767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764160.836769:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764160.836774:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3205 00000100:00080000:1.0:1787764160.836779:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764160.836785:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764160.836797:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274298752:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787764160.836797:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764160.836818:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274298880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764160.836882:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764160.836932:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764160/real 1787764160] req@ffff9ac447840000 x1874603274298752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764160.836952:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274298752/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764160.836965:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764160.836970:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764160.836974:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274298752:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764160.836999:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298880:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764160.837020:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764160.837032:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764160/real 1787764160] req@ffff9ac447840f00 x1874603274298880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764160.837046:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274298880/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764160.837056:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764160.837060:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764160.837064:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274298880:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764161.412734:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764161.860648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764162.436673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764162.884615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764163.460695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764163.908622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764164.484666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764164.932602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764165.508645:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764165.956627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764165.956642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764165.956648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764165.956652:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764165.956667:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3210 00000100:00080000:0.0:1787764165.956677:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764165.956685:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764165.956717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274299008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764165.956742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764165.956746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764165.956748:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764165.956752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3210 00000100:00080000:0.0:1787764165.956755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764165.956759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764165.956767:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764165.956779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274299136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764165.956789:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299008:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764165.956848:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764165.956887:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764165/real 1787764165] req@ffff9ac451f84000 x1874603274299008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764165.956899:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274299008/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764165.956906:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764165.956908:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764165.956911:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299008:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764165.956939:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299136:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764165.956955:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764165.956963:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764165/real 1787764165] req@ffff9ac451f870c0 x1874603274299136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764165.956970:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274299136/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764165.956978:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764165.956981:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764165.956984:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299136:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764166.532638:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764166.980585:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764167.556680:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764168.004643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764168.580649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764169.028639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764169.604704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764170.052650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764170.628655:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764171.076674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764171.076690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764171.076697:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764171.076701:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764171.076711:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3215 00000100:00080000:2.0:1787764171.076721:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764171.076728:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764171.076777:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274299264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764171.076809:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764171.076812:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764171.076814:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764171.076818:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3215 00000100:00080000:2.0:1787764171.076822:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764171.076840:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764171.076846:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299264:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787764171.076852:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764171.076867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274299392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764171.076910:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764171.076971:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764171/real 1787764171] req@ffff9ac4523bcf00 x1874603274299264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764171.076989:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274299264/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764171.077034:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764171.077054:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764171.077058:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299264:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764171.077081:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299392:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764171.077102:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764171.077113:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764171/real 1787764171] req@ffff9ac4523be580 x1874603274299392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764171.077122:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274299392/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764171.077129:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764171.077131:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764171.077134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299392:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764171.652673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764172.100627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764172.676695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764173.124641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764173.700701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764174.148640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764174.724741:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764175.172686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764175.748756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764176.196645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764176.196645:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764176.196657:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764176.196663:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764176.196675:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3220 00000100:00080000:2.0:1787764176.196690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764176.196700:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764176.196752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274299520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764176.196780:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764176.196783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764176.196785:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764176.196788:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3220 00000100:00080000:2.0:1787764176.196791:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764176.196795:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764176.196806:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764176.196815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274299648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764176.196832:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299520:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764176.196903:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764176.196946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764176/real 1787764176] req@ffff9ac4523bcb40 x1874603274299520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764176.196986:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274299520/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764176.196996:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764176.196999:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764176.197002:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299520:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764176.197042:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299648:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764176.197061:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764176.197070:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764176/real 1787764176] req@ffff9ac4523be1c0 x1874603274299648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764176.197079:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274299648/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764176.197087:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764176.197091:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764176.197095:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299648:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764176.772752:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764177.220632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764177.796708:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764178.244663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764178.820751:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764179.268652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764179.844732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764180.292623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764180.868775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787764181.316653:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787764181.316658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764181.316668:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764181.316697:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764181.316708:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3226 00000100:00080000:2.0:1787764181.316721:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764181.316729:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764181.316778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bf0c0 x1874603274299776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764181.316808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764181.316811:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764181.316813:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764181.316816:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3226 00000100:00080000:2.0:1787764181.316818:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764181.316821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764181.316831:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764181.316843:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be1c0 x1874603274299904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764181.316863:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299776:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764181.316944:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764181.317000:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764181/real 1787764181] req@ffff9ac4523bf0c0 x1874603274299776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764181.317015:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bf0c0 x1874603274299776/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764181.317024:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764181.317027:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764181.317030:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bf0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274299776:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764181.317055:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299904:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764181.317071:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764181.317079:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764181/real 1787764181] req@ffff9ac4523be1c0 x1874603274299904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764181.317107:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be1c0 x1874603274299904/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764181.317113:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764181.317115:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764181.317118:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274299904:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764181.892714:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764182.340643:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764182.916775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764183.365101:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764183.940688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764184.388632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764184.964775:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764185.412599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764185.988742:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764186.436604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764186.436632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764186.436638:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764186.436642:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764186.436652:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3231 00000100:00080000:3.0:1787764186.436663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764186.436680:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764186.436731:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274300032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764186.436756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764186.436758:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764186.436760:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764186.436763:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3231 00000100:00080000:3.0:1787764186.436766:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764186.436788:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764186.436798:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764186.436810:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274300160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764186.436816:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300032:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764186.436911:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764186.436980:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764186/real 1787764186] req@ffff9ac451c503c0 x1874603274300032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764186.437002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274300032/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764186.437014:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764186.437019:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764186.437024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300032:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764186.437050:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300160:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764186.437075:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764186.437088:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764186/real 1787764186] req@ffff9ac451c50780 x1874603274300160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764186.437102:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274300160/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764186.437126:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764186.437130:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764186.437134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300160:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764187.012780:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764187.460641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764188.036674:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764188.484663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764189.060755:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764189.508660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764190.084729:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764190.532631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764191.108773:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764191.556641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764191.556693:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764191.556699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764191.556704:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764191.556715:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3236 00000100:00080000:0.0:1787764191.556727:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764191.556734:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764191.556784:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274300288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764191.556811:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764191.556815:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764191.556816:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764191.556819:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3236 00000100:00080000:0.0:1787764191.556822:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764191.556826:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764191.556836:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764191.556846:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274300416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764191.556875:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300288:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764191.556958:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764191.557010:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764191/real 1787764191] req@ffff9ac451f861c0 x1874603274300288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764191.557027:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274300288/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764191.557036:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764191.557039:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764191.557060:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300288:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764191.557082:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300416:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764191.557103:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764191.557112:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764191/real 1787764191] req@ffff9ac451f84780 x1874603274300416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764191.557122:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274300416/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764191.557128:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764191.557131:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764191.557133:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300416:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764192.132703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764192.580632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764193.156734:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764193.604616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764194.180670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764194.628611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764195.204724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764195.652621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764196.228703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764196.676655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764196.676655:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764196.676667:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764196.676672:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764196.676683:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3241 00000100:00080000:3.0:1787764196.676694:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764196.676702:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764196.676743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274300544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764196.676768:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764196.676771:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764196.676772:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764196.676775:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3241 00000100:00080000:3.0:1787764196.676778:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764196.676780:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764196.676789:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764196.676799:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274300672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764196.676846:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300544:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764196.677015:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300672:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764197.252764:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764197.700693:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764198.276776:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764198.724701:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764199.300715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764199.748732:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787764199.748883:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787764199.748937:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787764199.748955:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787764199.749010:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787764199.749071:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764196/real 1787764199] req@ffff9ac451c50b40 x1874603274300672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764199.749133:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274300672/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764199.749146:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764199.749152:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764199.749158:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300672:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787764199.749167:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764196/real 1787764199] req@ffff9ac451c52940 x1874603274300544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764199.749182:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274300544/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764199.749191:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764199.749195:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764199.749198:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300544:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787764200.324731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764200.772771:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764201.348726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764201.796657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764201.796677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764201.796682:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764201.796695:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3246 00000100:00080000:3.0:1787764201.796713:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787764201.796720:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764201.796723:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764201.796768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274300800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764201.796814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764201.796818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764201.796820:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764201.796825:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3246 00000100:00080000:3.0:1787764201.796829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764201.796834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764201.796844:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764201.796858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274300928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764201.796977:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300800:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764201.797048:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764201.797095:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764201/real 1787764201] req@ffff9ac451c52d00 x1874603274300800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764201.797115:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274300800/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764201.797126:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764201.797130:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764201.797134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274300800:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764201.797160:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300928:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764201.797191:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764201.797205:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764201/real 1787764201] req@ffff9ac451c530c0 x1874603274300928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764201.797218:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274300928/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764201.797227:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764201.797230:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764201.797234:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274300928:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764202.372725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764202.820734:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764203.396663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764203.844625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764204.420711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764204.868686:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764205.444658:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764205.892635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764206.468730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787764206.916661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764206.916664:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764206.916678:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764206.916684:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764206.916703:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3251 00000100:00080000:0.0:1787764206.916715:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764206.916723:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764206.916771:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274301056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764206.916803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764206.916807:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764206.916809:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764206.916813:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3251 00000100:00080000:0.0:1787764206.916817:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764206.916838:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764206.916849:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764206.916867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274301184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764206.916867:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301056:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764206.916956:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764206.917008:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764206/real 1787764206] req@ffff9ac451f85680 x1874603274301056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764206.917026:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274301056/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764206.917047:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764206.917050:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764206.917053:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301056:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764206.917079:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301184:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764206.917097:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764206.917106:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764206/real 1787764206] req@ffff9ac451f843c0 x1874603274301184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764206.917114:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274301184/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764206.917120:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764206.917122:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764206.917124:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301184:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764207.492674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764207.940663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764208.516723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764208.964677:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764209.540724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764209.988658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764210.564686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764211.012647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764211.588751:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764212.036658:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764212.036659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764212.036668:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764212.036672:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764212.036680:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3256 00000100:00080000:3.0:1787764212.036690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764212.036697:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764212.036745:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274301312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764212.036764:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764212.036767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764212.036768:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764212.036771:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3256 00000100:00080000:3.0:1787764212.036774:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764212.036777:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764212.036786:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764212.036795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274301440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764212.036803:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301312:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764212.036870:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764212.036923:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764212/real 1787764212] req@ffff9ac451c530c0 x1874603274301312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764212.036943:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274301312/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764212.036967:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764212.036972:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764212.036976:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301312:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764212.037002:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301440:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764212.037027:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764212.037040:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764212/real 1787764212] req@ffff9ac451c52d00 x1874603274301440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764212.037053:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274301440/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764212.037063:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764212.037066:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764212.037069:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301440:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764212.612692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764213.060606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764213.636674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764214.084630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764214.660662:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764215.108663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764215.684722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764216.132636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764216.708723:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764217.156622:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764217.156630:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764217.156634:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787764217.156638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764217.156643:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3261 00000100:00080000:3.0:1787764217.156670:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764217.156676:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764217.156723:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274301568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764217.156742:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764217.156744:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764217.156746:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764217.156749:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3261 00000100:00080000:3.0:1787764217.156751:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764217.156755:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764217.156762:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764217.156771:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274301696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764217.156780:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301568:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764217.156838:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764217.156884:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764217/real 1787764217] req@ffff9ac451c52940 x1874603274301568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764217.156898:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274301568/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764217.156917:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764217.156920:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764217.156924:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301568:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764217.156945:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301696:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764217.156964:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764217.156973:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764217/real 1787764217] req@ffff9ac451c50b40 x1874603274301696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764217.156982:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274301696/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764217.156988:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764217.156990:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764217.156992:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301696:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764217.732753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764218.180633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764218.756698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764219.204633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764219.780738:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764220.228647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764220.804687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764221.252588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764221.828722:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764222.276641:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764222.276660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764222.276664:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764222.276672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3267 00000100:00080000:3.0:1787764222.276682:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764222.276689:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764222.276721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274301824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764222.276746:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764222.276748:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764222.276765:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764222.276766:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787764222.276794:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301824:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764222.276857:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764222.276895:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764222/real 1787764222] req@ffff9ac451c50780 x1874603274301824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764222.276908:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274301824/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764222.276916:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764222.276919:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764222.276922:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274301824:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787764222.276973:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3267 00000100:00080000:3.0:1787764222.276979:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764222.276983:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764222.276992:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764222.277004:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274301952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764222.277049:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301952:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764222.277071:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764222.277081:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764222/real 1787764222] req@ffff9ac451c503c0 x1874603274301952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764222.277090:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274301952/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764222.277097:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764222.277099:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764222.277102:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274301952:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764222.852632:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764223.300604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764223.876700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764224.324703:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764224.900664:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764225.348610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764225.924716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764226.372694:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764226.948702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764227.396640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764227.396651:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764227.396659:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764227.396663:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764227.396673:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3272 00000100:00080000:0.0:1787764227.396683:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764227.396690:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764227.396721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274302080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764227.396745:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764227.396747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764227.396749:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764227.396753:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3272 00000100:00080000:0.0:1787764227.396757:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764227.396762:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764227.396771:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302080:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787764227.396772:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764227.396792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274302208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764227.396832:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764227.396889:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764227/real 1787764227] req@ffff9ac451f843c0 x1874603274302080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764227.396903:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274302080/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764227.396911:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764227.396915:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764227.396918:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302080:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764227.396935:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302208:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764227.396955:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764227.396969:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764227/real 1787764227] req@ffff9ac451f85680 x1874603274302208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764227.396983:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274302208/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764227.396993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764227.396996:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764227.397000:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302208:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764227.972685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764228.420645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764228.996670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764229.444661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764230.020675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764230.468614:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764231.044670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764231.492640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764232.068704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764232.516630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764232.516637:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764232.516643:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764232.516647:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764232.516655:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3277 00000100:00080000:0.0:1787764232.516673:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764232.516679:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764232.516711:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274302336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764232.516742:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764232.516745:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764232.516746:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764232.516750:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3277 00000100:00080000:0.0:1787764232.516753:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764232.516757:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764232.516766:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764232.516771:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764232.516777:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274302464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764232.516827:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764232.516863:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764232/real 1787764232] req@ffff9ac451f84780 x1874603274302336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764232.516876:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274302336/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764232.516884:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764232.516888:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764232.516890:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302336:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764232.516908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302464:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764232.516927:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764232.516939:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764232/real 1787764232] req@ffff9ac451f861c0 x1874603274302464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764232.516953:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274302464/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764232.516971:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764232.516975:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764232.516979:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302464:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764233.092682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764233.540656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764234.116650:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764234.564638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764235.140657:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764235.588650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764236.164736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764236.612649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764237.188659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764237.636587:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764237.636615:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764237.636621:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764237.636624:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764237.636633:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3282 00000100:00080000:0.0:1787764237.636642:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764237.636649:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764237.636681:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274302592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764237.636702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764237.636704:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764237.636706:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764237.636725:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3282 00000100:00080000:0.0:1787764237.636727:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764237.636731:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764237.636748:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302592:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787764237.636753:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764237.636766:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274302720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764237.636821:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764237.636868:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764237/real 1787764237] req@ffff9ac451f870c0 x1874603274302592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764237.636883:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274302592/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764237.636890:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764237.636893:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764237.636896:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302592:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764237.636915:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302720:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764237.636931:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764237.636940:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764237/real 1787764237] req@ffff9ac451f84000 x1874603274302720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764237.636948:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274302720/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764237.636954:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764237.636956:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764237.636958:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302720:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764238.212695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764238.660637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764239.236685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764239.684639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764240.260681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764240.708624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764241.284784:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764241.732640:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764242.308690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764242.756626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764242.756637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1787764242.756642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764242.756643:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764242.756657:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3287 00000100:00080000:3.0:1787764242.756667:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764242.756674:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764242.756706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274302848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764242.756752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764242.756754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764242.756756:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764242.756759:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3287 00000100:00080000:3.0:1787764242.756762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764242.756766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764242.756774:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764242.756789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274302976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764242.756790:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302848:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764242.756859:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764242.756910:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764242/real 1787764242] req@ffff9ac451c50f00 x1874603274302848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764242.756926:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274302848/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764242.756935:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764242.756938:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764242.756941:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274302848:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764242.756966:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302976:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764242.756983:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764242.756993:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764242/real 1787764242] req@ffff9ac451c52580 x1874603274302976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764242.757002:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274302976/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764242.757008:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764242.757010:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764242.757012:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274302976:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764243.332699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764243.780659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764244.356663:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764244.804658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764245.380750:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764245.828661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764246.404675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764246.852606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764247.428720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764247.876637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764247.876650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764247.876658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764247.876662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764247.876672:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3292 00000100:00080000:0.0:1787764247.876683:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764247.876691:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764247.876751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274303104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764247.876778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764247.876781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764247.876782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764247.876786:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3292 00000100:00080000:0.0:1787764247.876790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764247.876794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764247.876805:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764247.876808:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764247.876821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274303232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764247.876885:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764247.876948:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764247/real 1787764247] req@ffff9ac451f87c00 x1874603274303104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764247.876962:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274303104/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764247.876972:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764247.876977:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764247.876981:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303104:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764247.877010:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274303232:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764247.877034:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764247.877044:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764247/real 1787764247] req@ffff9ac451f84b40 x1874603274303232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764247.877055:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274303232/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764247.877064:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764247.877067:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764247.877071:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274303232:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764248.452733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764248.900665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764249.476735:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764249.924638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764250.500703:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764250.948638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764251.524685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764251.972616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764252.548691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764252.996631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764252.996640:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764252.996646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764252.996650:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764252.996658:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3297 00000100:00080000:1.0:1787764252.996667:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764252.996674:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764252.996705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274303360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764252.996726:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764252.996728:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764252.996730:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764252.996733:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3297 00000100:00080000:1.0:1787764252.996749:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787764252.996754:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303360:192.168.203.123@tcp:250:kworker.0 00000100:00080000:1.0:1787764252.996755:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764252.996767:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764252.996781:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274303488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764252.996816:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764252.996861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764252/real 1787764252] req@ffff9ac447841680 x1874603274303360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764252.996876:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274303360/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764252.996884:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764252.996888:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764252.996890:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303360:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764252.996908:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274303488:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764252.996936:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764252.996945:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764252/real 1787764252] req@ffff9ac447840780 x1874603274303488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764252.996953:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274303488/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764252.996960:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764252.996962:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764252.996964:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274303488:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764253.572677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764254.020597:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764254.596646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764255.044642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764255.620671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764256.068626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764256.644667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764257.092635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764257.668659:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764258.116602:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764258.116602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764258.116611:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764258.116615:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764258.116623:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3302 00000100:00080000:1.0:1787764258.116632:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764258.116638:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764258.116669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274303616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764258.116690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764258.116693:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764258.116694:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764258.116697:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3302 00000100:00080000:1.0:1787764258.116699:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764258.116703:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764258.116710:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764258.116718:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274303744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764258.116725:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303616:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764258.116780:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764258.116831:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764258/real 1787764258] req@ffff9ac4478403c0 x1874603274303616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764258.116846:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274303616/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764258.116858:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764258.116862:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764258.116866:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303616:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764258.116890:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274303744:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764258.116913:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764258.116923:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764258/real 1787764258] req@ffff9ac4478430c0 x1874603274303744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764258.116932:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274303744/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764258.116938:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764258.116940:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764258.116942:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274303744:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764258.692678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764259.140593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764259.716733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764260.164627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764260.740750:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764261.188623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764261.764673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764262.212605:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764262.788700:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764263.236653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764263.236668:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764263.236680:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764263.236684:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764263.236693:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3307 00000100:00080000:0.0:1787764263.236719:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764263.236725:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764263.236754:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274303872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764263.236780:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764263.236783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764263.236784:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764263.236788:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3307 00000100:00080000:0.0:1787764263.236790:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764263.236794:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764263.236802:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764263.236815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274304000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764263.236822:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303872:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764263.236946:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304000:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764263.812692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764264.260602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764264.836787:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764265.284626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764265.860691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764266.308635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787764266.308815:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787764266.308860:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787764266.308879:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787764266.308942:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:2.0:1787764266.309076:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764263/real 1787764266] req@ffff9ac451f87c00 x1874603274304000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764266.309109:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274304000/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764266.309123:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764266.309129:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764266.309134:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304000:192.168.203.123@tcp:38:kworker.0 00000100:00100000:2.0:1787764266.309144:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764263/real 1787764266] req@ffff9ac451f84b40 x1874603274303872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764266.309159:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274303872/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764266.309169:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764266.309173:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764266.309177:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274303872:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787764266.884723:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764267.332634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764267.908641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764268.356633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764268.356635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764268.356644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764268.356648:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764268.356659:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3313 00000100:00080000:0.0:1787764268.356670:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764268.356678:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764268.356712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274304128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764268.356740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764268.356744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764268.356759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764268.356764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3313 00000100:00080000:0.0:1787764268.356768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787764268.356770:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304128:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787764268.356775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764268.356785:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764268.356797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274304256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764268.356832:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764268.356882:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764268/real 1787764268] req@ffff9ac451f84000 x1874603274304128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764268.356923:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274304128/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764268.356935:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764268.356939:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764268.356944:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304128:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764268.356967:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304256:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764268.356993:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764268.357008:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764268/real 1787764268] req@ffff9ac451f870c0 x1874603274304256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764268.357022:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274304256/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764268.357031:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764268.357035:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764268.357039:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304256:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764268.932667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764269.380603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764269.956648:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764270.404652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764270.980662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764271.428679:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764272.004647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764272.452649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764273.028646:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764273.476611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764273.476618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764273.476624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764273.476628:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764273.476636:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3318 00000100:00080000:0.0:1787764273.476646:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764273.476652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764273.476684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274304384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764273.476704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764273.476707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764273.476709:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764273.476712:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3318 00000100:00080000:0.0:1787764273.476715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764273.476718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764273.476727:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764273.476734:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304384:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764273.476738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274304512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764273.476789:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764273.476827:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764273/real 1787764273] req@ffff9ac451f861c0 x1874603274304384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764273.476848:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274304384/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764273.476856:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764273.476859:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764273.476862:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304384:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764273.476880:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304512:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764273.476897:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764273.476906:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764273/real 1787764273] req@ffff9ac451f84780 x1874603274304512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764273.476914:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274304512/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764273.476920:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764273.476922:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764273.476924:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304512:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764274.052660:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764274.500635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764275.076713:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764275.524603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764276.100758:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764276.548621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764277.124797:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764277.572650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764278.148698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764278.596634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764278.596659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764278.596665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764278.596669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764278.596679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3323 00000100:00080000:0.0:1787764278.596691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764278.596699:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764278.596734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274304640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764278.596757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764278.596761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764278.596763:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764278.596767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3323 00000100:00080000:0.0:1787764278.596772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764278.596776:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764278.596786:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764278.596791:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304640:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764278.596800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274304768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764278.596852:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764278.596885:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764278/real 1787764278] req@ffff9ac451f85680 x1874603274304640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764278.596911:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274304640/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764278.596919:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764278.596922:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764278.596925:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304640:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764278.596958:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304768:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764278.596976:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764278.596986:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764278/real 1787764278] req@ffff9ac451f843c0 x1874603274304768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764278.596994:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274304768/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764278.597000:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764278.597003:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764278.597005:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274304768:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764279.172747:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764279.620604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764280.196635:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764280.644603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764281.220687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764281.668592:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764282.244683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764282.692630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764283.268675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764283.716612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764283.716629:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764283.716638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764283.716643:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764283.716653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3328 00000100:00080000:0.0:1787764283.716664:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764283.716672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764283.716716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274304896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764283.716738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764283.716740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764283.716742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764283.716746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3328 00000100:00080000:0.0:1787764283.716749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764283.716754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764283.716762:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764283.716768:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764283.716775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274305024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764283.716829:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764283.716887:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764283/real 1787764283] req@ffff9ac451f843c0 x1874603274304896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764283.716908:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274304896/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764283.716916:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764283.716920:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764283.716923:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274304896:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764283.716941:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305024:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764283.716959:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764283.716969:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764283/real 1787764283] req@ffff9ac451f85680 x1874603274305024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764283.716977:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274305024/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764283.716983:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764283.716985:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764283.716988:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305024:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764284.292719:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764284.740636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764285.316763:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764285.764594:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764286.340692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764286.788660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764287.364653:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764287.812603:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764288.388730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764288.836661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764288.836720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764288.836728:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764288.836732:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764288.836742:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3333 00000100:00080000:1.0:1787764288.836755:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764288.836762:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764288.836801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274305152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764288.836826:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764288.836829:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764288.836830:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764288.836833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3333 00000100:00080000:1.0:1787764288.836836:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764288.836840:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764288.836848:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764288.836857:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274305280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764288.836876:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305152:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764288.836955:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764288.837019:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764288/real 1787764288] req@ffff9ac4478421c0 x1874603274305152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764288.837053:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274305152/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764288.837070:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764288.837074:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764288.837078:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305152:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764288.837101:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305280:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764288.837120:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764288.837130:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764288/real 1787764288] req@ffff9ac447842d00 x1874603274305280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764288.837138:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274305280/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764288.837144:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764288.837147:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764288.837150:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305280:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764289.412711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764289.860629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764290.436688:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764290.884662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764291.460753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764291.908658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764292.484684:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764292.932618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764293.508753:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764293.956653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764293.956656:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764293.956667:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764293.956672:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764293.956685:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3338 00000100:00080000:0.0:1787764293.956699:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764293.956709:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764293.956767:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274305408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764293.956799:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764293.956803:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764293.956805:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764293.956811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3338 00000100:00080000:0.0:1787764293.956816:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787764293.956838:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305408:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787764293.956845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764293.956860:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764293.956879:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274305536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764293.956925:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764293.956982:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764293/real 1787764293] req@ffff9ac451f84780 x1874603274305408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764293.957012:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274305408/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764293.957025:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764293.957031:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764293.957036:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305408:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764293.957063:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305536:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764293.957088:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764293.957101:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764293/real 1787764293] req@ffff9ac451f861c0 x1874603274305536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764293.957114:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274305536/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764293.957124:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764293.957127:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764293.957131:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305536:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764294.532703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764294.980598:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764295.556724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764296.004737:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764296.580689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764297.028647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764297.604687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764298.052645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764298.628669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764299.076604:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764299.076628:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764299.076637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764299.076642:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764299.076652:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3343 00000100:00080000:1.0:1787764299.076663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764299.076682:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764299.076717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274305664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764299.076750:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764299.076755:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764299.076757:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764299.076763:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3343 00000100:00080000:1.0:1787764299.076767:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764299.076772:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764299.076784:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764299.076798:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764299.076804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274305792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764299.076862:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764299.076916:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764299/real 1787764299] req@ffff9ac447840b40 x1874603274305664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764299.076944:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274305664/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764299.076953:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764299.076956:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764299.076959:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305664:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764299.076977:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305792:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764299.076997:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764299.077006:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764299/real 1787764299] req@ffff9ac447842940 x1874603274305792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764299.077015:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274305792/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764299.077022:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764299.077024:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764299.077027:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274305792:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764299.652672:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764300.100633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764300.676669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764301.124639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764301.700727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764302.148647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764302.724641:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764303.172630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764303.748680:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764304.196610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764304.196626:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764304.196632:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764304.196636:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764304.196644:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3348 00000100:00080000:0.0:1787764304.196653:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764304.196659:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764304.196689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274305920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764304.196710:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764304.196713:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764304.196714:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764304.196717:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3348 00000100:00080000:0.0:1787764304.196720:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764304.196723:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764304.196730:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764304.196739:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274306048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764304.196743:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305920:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764304.196803:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764304.196834:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764304/real 1787764304] req@ffff9ac451f870c0 x1874603274305920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764304.196861:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274305920/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764304.196869:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764304.196872:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764304.196875:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274305920:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764304.196895:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306048:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764304.196914:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764304.196927:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764304/real 1787764304] req@ffff9ac451f84000 x1874603274306048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764304.196940:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274306048/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764304.196948:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764304.196951:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764304.196954:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306048:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764304.772690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764305.220653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764305.796686:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764306.244642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764306.820656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764307.268663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764307.844708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764308.292627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764308.868716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764309.316659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764309.316679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764309.316686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764309.316690:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764309.316701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3354 00000100:00080000:3.0:1787764309.316714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764309.316722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764309.316791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274306176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764309.316818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764309.316822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764309.316824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764309.316829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3354 00000100:00080000:3.0:1787764309.316834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764309.316839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764309.316851:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764309.316856:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306176:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787764309.316870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274306304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764309.316927:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764309.316975:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764309/real 1787764309] req@ffff9ac451c52580 x1874603274306176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764309.317000:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274306176/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764309.317009:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764309.317013:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764309.317016:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306176:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764309.317040:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306304:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764309.317076:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764309.317086:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764309/real 1787764309] req@ffff9ac451c50f00 x1874603274306304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764309.317095:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274306304/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764309.317101:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764309.317103:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764309.317106:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306304:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764309.892691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764310.340628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764310.916695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764311.364636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764311.940660:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764312.388630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764312.964696:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764313.412638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764313.988690:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764314.436611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764314.436612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764314.436620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764314.436624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764314.436632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3359 00000100:00080000:3.0:1787764314.436642:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764314.436649:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764314.436680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274306432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764314.436699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764314.436702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764314.436703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764314.436706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3359 00000100:00080000:3.0:1787764314.436709:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764314.436725:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764314.436729:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306432:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787764314.436742:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764314.436758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274306560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764314.436796:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764314.436838:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764314/real 1787764314] req@ffff9ac451c503c0 x1874603274306432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764314.436872:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274306432/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764314.436883:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764314.436888:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764314.436892:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306432:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764314.436917:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306560:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764314.436937:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764314.436947:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764314/real 1787764314] req@ffff9ac451c50780 x1874603274306560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764314.436956:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274306560/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764314.436962:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764314.436964:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764314.436967:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306560:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764315.012716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764315.460632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764316.036656:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764316.484627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764317.060712:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764317.508658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764318.084677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764318.532631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764319.108722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764319.556651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764319.556654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764319.556663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764319.556668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764319.556679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3364 00000100:00080000:3.0:1787764319.556691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764319.556699:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764319.556746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274306688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764319.556775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764319.556778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764319.556779:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764319.556782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3364 00000100:00080000:3.0:1787764319.556785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764319.556788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764319.556800:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764319.556807:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787764319.556812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274306816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764319.556877:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764319.556933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764319/real 1787764319] req@ffff9ac451c50b40 x1874603274306688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764319.556964:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274306688/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764319.556987:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764319.556991:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764319.556994:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306688:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764319.557019:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306816:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764319.557038:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764319.557047:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764319/real 1787764319] req@ffff9ac451c52940 x1874603274306816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764319.557055:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274306816/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764319.557061:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764319.557063:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764319.557065:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274306816:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764320.132689:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764320.580649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764321.156720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764321.604626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764322.180647:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764322.628617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764323.204653:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764323.652651:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764324.228716:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787764324.676609:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764324.676621:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1787764324.676625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764324.676626:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764324.676642:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3369 00000100:00080000:2.0:1787764324.676677:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764324.676687:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764324.676733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcb40 x1874603274306944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764324.676762:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764324.676765:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764324.676766:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764324.676770:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3369 00000100:00080000:2.0:1787764324.676773:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764324.676776:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764324.676787:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764324.676798:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be580 x1874603274307072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764324.676810:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306944:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764324.676892:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764324.676966:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764324/real 1787764324] req@ffff9ac4523bcb40 x1874603274306944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764324.676987:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcb40 x1874603274306944/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764324.676999:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764324.677004:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764324.677008:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274306944:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764324.677037:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307072:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764324.677065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764324.677080:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764324/real 1787764324] req@ffff9ac4523be580 x1874603274307072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764324.677093:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be580 x1874603274307072/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764324.677103:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764324.677107:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764324.677111:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307072:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764325.252801:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764325.701015:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764326.276701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764326.724617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764327.300707:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764327.748656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764328.324673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764328.772647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764329.348717:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764329.796651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764329.796659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764329.796662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787764329.796667:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764329.796672:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3374 00000100:00080000:3.0:1787764329.796685:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764329.796691:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764329.796720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274307200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764329.796740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764329.796743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764329.796744:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764329.796747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3374 00000100:00080000:3.0:1787764329.796749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764329.796751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764329.796759:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764329.796768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274307328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764329.796793:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307200:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764329.796939:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307328:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764330.372729:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764330.820631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764331.396706:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764331.844628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764332.420692:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764332.868632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787764332.868804:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787764332.868853:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787764332.868869:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787764332.868918:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787764332.868955:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764329/real 1787764332] req@ffff9ac451c530c0 x1874603274307328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764332.868979:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274307328/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764332.868990:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764332.868994:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764332.868997:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307328:192.168.203.123@tcp:38:kworker.0 00000100:00100000:0.0:1787764332.869004:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764329/real 1787764332] req@ffff9ac451c52d00 x1874603274307200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764332.869017:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274307200/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764332.869025:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764332.869029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764332.869032:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307200:192.168.203.123@tcp:250:kworker.0 00010000:00010000:1.0:1787764333.444673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787764333.892668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764334.468679:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764334.916658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764334.916686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764334.916691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764334.916695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764334.916702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3379 00000100:00080000:0.0:1787764334.916712:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764334.916718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764334.916742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274307456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764334.916769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764334.916771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764334.916773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764334.916777:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3379 00000100:00080000:0.0:1787764334.916779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764334.916782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764334.916789:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307456:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787764334.916790:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764334.916801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274307584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764334.916843:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764334.916902:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764334/real 1787764334] req@ffff9ac451f84b40 x1874603274307456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764334.916922:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274307456/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764334.916936:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764334.916939:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764334.916942:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307456:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764334.916973:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307584:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764334.916996:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764334.917008:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764334/real 1787764334] req@ffff9ac451f87c00 x1874603274307584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764334.917019:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274307584/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764334.917025:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764334.917028:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764334.917031:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307584:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764335.492669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764335.940608:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764336.516673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764336.964633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764337.540656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764337.988648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764338.564681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764339.012630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764339.588689:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764340.036611:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764340.036612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764340.036621:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764340.036626:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764340.036633:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3384 00000100:00080000:0.0:1787764340.036641:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764340.036648:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764340.036676:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274307712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764340.036695:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764340.036697:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764340.036699:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764340.036702:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3384 00000100:00080000:0.0:1787764340.036704:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764340.036707:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764340.036722:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307712:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787764340.036727:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764340.036743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274307840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764340.036769:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764340.036804:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764340/real 1787764340] req@ffff9ac451f87c00 x1874603274307712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764340.036816:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274307712/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764340.036828:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764340.036843:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764340.036846:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307712:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764340.036863:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307840:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764340.036880:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764340.036888:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764340/real 1787764340] req@ffff9ac451f84b40 x1874603274307840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764340.036896:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274307840/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764340.036901:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764340.036903:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764340.036905:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274307840:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764340.612790:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764341.060652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764341.636624:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764342.084619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764342.660693:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764343.108635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764343.684669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764344.132609:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764344.708657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764345.156629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764345.156642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764345.156648:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764345.156652:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764345.156661:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3389 00000100:00080000:3.0:1787764345.156671:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764345.156677:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764345.156711:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52d00 x1874603274307968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764345.156748:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764345.156752:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764345.156753:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764345.156758:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3389 00000100:00080000:3.0:1787764345.156762:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764345.156767:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764345.156777:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764345.156782:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787764345.156808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c530c0 x1874603274308096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764345.156850:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764345.156885:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764345/real 1787764345] req@ffff9ac451c52d00 x1874603274307968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764345.156898:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52d00 x1874603274307968/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764345.156911:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764345.156914:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764345.156917:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274307968:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764345.156940:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308096:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764345.156956:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764345.156964:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764345/real 1787764345] req@ffff9ac451c530c0 x1874603274308096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764345.156971:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c530c0 x1874603274308096/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764345.156976:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764345.156979:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764345.156980:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c530c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308096:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764345.732661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764346.180588:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764346.756673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764347.204602:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764347.780670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764348.228636:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764348.804720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764349.252684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764349.828725:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787764350.276646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764350.276649:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764350.276656:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764350.276661:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764350.276670:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3395 00000100:00080000:3.0:1787764350.276681:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764350.276688:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764350.276729:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52940 x1874603274308224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764350.276772:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764350.276777:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764350.276779:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764350.276786:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3395 00000100:00100000:2.0:1787764350.276787:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308224:192.168.203.123@tcp:250:kworker.0 00000100:00080000:3.0:1787764350.276791:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764350.276798:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764350.276829:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764350.276844:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50b40 x1874603274308352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764350.276855:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764350.276895:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764350/real 1787764350] req@ffff9ac451c52940 x1874603274308224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764350.276908:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52940 x1874603274308224/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764350.276920:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764350.276924:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764350.276926:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308224:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764350.276949:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308352:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764350.276965:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764350.276982:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764350/real 1787764350] req@ffff9ac451c50b40 x1874603274308352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764350.276989:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50b40 x1874603274308352/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764350.276994:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764350.276996:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764350.276998:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308352:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764350.852711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764351.300633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764351.876692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764352.324625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764352.900687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764353.348634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764353.924720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764354.372637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764354.948657:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764355.396616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764355.396662:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764355.396675:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764355.396681:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764355.396694:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3400 00000100:00080000:3.0:1787764355.396708:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764355.396717:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764355.396780:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50780 x1874603274308480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764355.396814:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764355.396818:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764355.396820:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764355.396826:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3400 00000100:00080000:3.0:1787764355.396831:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764355.396836:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764355.396849:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308480:192.168.203.123@tcp:250:kworker.0 00000008:00100000:3.0:1787764355.396849:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787764355.396882:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c503c0 x1874603274308608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764355.396924:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764355.396967:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764355/real 1787764355] req@ffff9ac451c50780 x1874603274308480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764355.396987:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50780 x1874603274308480/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764355.397009:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764355.397013:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764355.397016:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308480:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764355.397036:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308608:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764355.397055:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764355.397063:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764355/real 1787764355] req@ffff9ac451c503c0 x1874603274308608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764355.397072:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c503c0 x1874603274308608/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764355.397077:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764355.397080:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764355.397082:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c503c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308608:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764355.972736:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764356.420629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764356.996665:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764357.444599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764358.020722:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764358.468663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764359.044661:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764359.492610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764360.068682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764360.516618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764360.516633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764360.516640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764360.516644:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764360.516653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3405 00000100:00080000:0.0:1787764360.516662:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764360.516669:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764360.516712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274308736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764360.516738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764360.516741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764360.516743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764360.516748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3405 00000100:00080000:0.0:1787764360.516751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764360.516756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764360.516766:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764360.516771:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308736:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764360.516781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274308864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764360.516830:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764360.516860:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764360/real 1787764360] req@ffff9ac451f84000 x1874603274308736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764360.516872:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274308736/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764360.516890:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764360.516893:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764360.516895:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308736:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764360.516912:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308864:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764360.516928:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764360.516936:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764360/real 1787764360] req@ffff9ac451f870c0 x1874603274308864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764360.516943:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274308864/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764360.516948:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764360.516950:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764360.516952:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274308864:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764361.092643:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764361.540644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764362.116715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764362.564659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764363.140710:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764363.588615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764364.164629:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764364.612617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764365.188631:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764365.636613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787764365.636638:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764365.636645:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787764365.636649:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764365.636672:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3410 00000100:00080000:3.0:1787764365.636698:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764365.636706:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:3.0:1787764365.636736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c50f00 x1874603274308992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787764365.636761:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787764365.636765:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787764365.636766:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787764365.636770:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3410 00000100:00080000:3.0:1787764365.636774:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787764365.636778:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:3.0:1787764365.636787:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764365.636796:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:3.0:1787764365.636801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451c52580 x1874603274309120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764365.636852:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764365.636897:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764365/real 1787764365] req@ffff9ac451c50f00 x1874603274308992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764365.636916:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c50f00 x1874603274308992/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764365.636932:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764365.636936:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764365.636940:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c50f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274308992:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764365.636961:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309120:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764365.636981:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764365.636991:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764365/real 1787764365] req@ffff9ac451c52580 x1874603274309120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764365.637003:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451c52580 x1874603274309120/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764365.637011:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764365.637014:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764365.637017:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451c52580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309120:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764366.212637:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764366.660658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764367.236682:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764367.684616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764368.260670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764368.708623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764369.284704:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764369.732618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764370.308676:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764370.756637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764370.756641:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764370.756649:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764370.756653:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764370.756663:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3415 00000100:00080000:0.0:1787764370.756685:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764370.756693:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764370.756724:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274309248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764370.756747:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764370.756751:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764370.756753:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764370.756758:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3415 00000100:00080000:0.0:1787764370.756762:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764370.756766:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764370.756775:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764370.756783:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274309248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764370.756789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274309376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764370.756843:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764370.756878:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764370/real 1787764370] req@ffff9ac451f861c0 x1874603274309248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764370.756903:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274309248/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764370.756921:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764370.756925:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764370.756929:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274309248:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764370.756952:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309376:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764370.756979:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764370.756994:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764370/real 1787764370] req@ffff9ac451f84780 x1874603274309376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764370.757008:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274309376/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764370.757017:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764370.757020:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764370.757024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309376:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764371.332701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764371.780653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764372.356661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764372.804623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764373.380701:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764373.828637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764374.404705:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764374.852623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764375.428714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764375.876654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764375.876658:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764375.876666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764375.876672:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764375.876685:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3420 00000100:00080000:0.0:1787764375.876698:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764375.876707:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764375.876748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274309504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764375.876773:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764375.876776:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764375.876793:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764375.876797:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3420 00000100:00080000:0.0:1787764375.876800:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764375.876804:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764375.876813:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764375.876819:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274309504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764375.876825:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274309632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787764375.876887:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764375.876934:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764375/real 1787764375] req@ffff9ac451f85680 x1874603274309504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764375.876948:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274309504/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764375.876962:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764375.876965:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764375.876968:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274309504:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764375.876989:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309632:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764375.877006:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764375.877013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764375/real 1787764375] req@ffff9ac451f843c0 x1874603274309632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764375.877021:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274309632/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764375.877026:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764375.877028:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764375.877030:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309632:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764376.452679:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764376.900647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764377.476733:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764377.924642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764378.500733:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764378.948611:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764379.524769:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764379.972685:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764380.548704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764380.996647:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787764380.996653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764380.996666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764380.996672:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764380.996684:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3425 00000100:00080000:0.0:1787764380.996696:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764380.996705:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764380.996748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274309760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764380.996772:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764380.996775:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764380.996776:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764380.996779:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3425 00000100:00080000:0.0:1787764380.996781:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764380.996783:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764380.996792:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764380.996802:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274309888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764380.996818:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274309760:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764380.996895:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764380.996946:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764380/real 1787764380] req@ffff9ac451f843c0 x1874603274309760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764380.996963:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274309760/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764380.996980:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764380.996984:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764380.996987:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274309760:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764380.997013:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309888:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764380.997034:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764380.997043:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764380/real 1787764380] req@ffff9ac451f85680 x1874603274309888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764380.997053:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274309888/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764380.997060:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764380.997062:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764380.997065:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274309888:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764381.572800:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764382.020655:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764382.596647:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764383.044619:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764383.620724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764384.068673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764384.644673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764385.092656:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764385.668726:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764386.116654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764386.116665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764386.116671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764386.116675:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764386.116685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3430 00000100:00080000:1.0:1787764386.116695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764386.116702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764386.116740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843c00 x1874603274310016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764386.116767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764386.116769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764386.116770:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764386.116773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3430 00000100:00080000:1.0:1787764386.116775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764386.116797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764386.116806:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764386.116820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274310144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764386.116820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310016:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764386.116891:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764386.116951:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764386/real 1787764386] req@ffff9ac447843c00 x1874603274310016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764386.116970:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843c00 x1874603274310016/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764386.116993:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764386.116997:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764386.117006:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310016:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764386.117028:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310144:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764386.117046:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764386.117055:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764386/real 1787764386] req@ffff9ac447842940 x1874603274310144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764386.117063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274310144/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764386.117068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764386.117070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764386.117072:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310144:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764386.692661:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764387.140627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764387.716676:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764388.164639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764388.740667:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764389.188593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764389.764662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764390.212620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764390.788702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764391.236612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764391.236635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764391.236640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764391.236645:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764391.236653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3435 00000100:00080000:0.0:1787764391.236663:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764391.236669:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764391.236706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274310272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764391.236727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764391.236729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764391.236731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764391.236736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3435 00000100:00080000:0.0:1787764391.236738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764391.236744:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764391.236753:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764391.236762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274310400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764391.236769:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310272:192.168.203.123@tcp:250:kworker.0 00000800:00000100:2.0:1787764391.236840:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764391.236885:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764391/real 1787764391] req@ffff9ac451f84780 x1874603274310272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764391.236899:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274310272/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764391.236907:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764391.236911:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764391.236914:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310272:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764391.236950:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310400:192.168.203.123@tcp:38:kworker.0 00000800:00000100:2.0:1787764391.236967:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:2.0:1787764391.236974:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764391/real 1787764391] req@ffff9ac451f861c0 x1874603274310400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787764391.236981:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274310400/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787764391.236986:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787764391.236988:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:2.0:1787764391.236990:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310400:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764391.812659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764392.260638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764392.836756:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764393.284674:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764393.860714:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764394.308616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764394.884716:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764395.332666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764395.908670:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764396.356622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764396.356628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764396.356636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764396.356639:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764396.356650:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3441 00000100:00080000:0.0:1787764396.356660:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764396.356666:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764396.356718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274310528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764396.356742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764396.356744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764396.356746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764396.356749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3441 00000100:00080000:0.0:1787764396.356751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764396.356754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764396.356762:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764396.356771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274310656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787764396.356790:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:2.0:1787764396.356881:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310656:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764396.932687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764397.380633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764397.956697:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764398.404600:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764398.980721:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764399.428637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787764399.428791:0:2332:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:2.0:1787764399.428831:0:2332:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787764399.428847:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787764399.428884:0:2332:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787764399.429346:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764396/real 1787764399] req@ffff9ac451f870c0 x1874603274310528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764399.429371:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274310528/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764399.429384:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764399.429389:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764399.429394:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310528:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764399.429402:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764396/real 1787764399] req@ffff9ac451f84000 x1874603274310656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764399.429415:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274310656/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764399.429460:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764399.429465:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764399.429474:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310656:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764400.004668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764400.452610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764401.028739:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764401.476653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764401.476747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764401.476758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764401.476763:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764401.476776:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3446 00000100:00080000:1.0:1787764401.476788:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764401.476797:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764401.476844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274310784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764401.476875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764401.476898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764401.476901:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764401.476907:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3446 00000100:00080000:1.0:1787764401.476912:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764401.476918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764401.476929:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764401.476931:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310784:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764401.476945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274310912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764401.477026:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764401.477070:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764401/real 1787764401] req@ffff9ac447840b40 x1874603274310784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764401.477088:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274310784/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764401.477099:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764401.477103:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764401.477107:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274310784:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764401.477131:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310912:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764401.477150:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764401.477160:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764401/real 1787764401] req@ffff9ac447842d00 x1874603274310912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764401.477171:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274310912/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764401.477179:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764401.477181:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764401.477190:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274310912:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764402.052664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764402.500647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764403.076691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764403.524653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764404.100685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764404.548638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764405.124693:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764405.572648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764406.148648:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764406.596668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764406.596668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764406.596681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764406.596686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764406.596696:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3451 00000100:00080000:0.0:1787764406.596707:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764406.596715:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764406.596747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274311040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764406.596777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764406.596781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764406.596783:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764406.596789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3451 00000100:00080000:0.0:1787764406.596793:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764406.596798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764406.596803:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311040:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787764406.596817:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764406.596836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274311168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787764406.596855:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764406.596884:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764406/real 1787764406] req@ffff9ac451f84000 x1874603274311040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764406.596903:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274311040/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764406.596913:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764406.596918:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764406.596922:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311040:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764406.596940:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311168:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764406.596957:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764406.596978:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764406/real 1787764406] req@ffff9ac451f870c0 x1874603274311168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764406.596985:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274311168/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764406.596991:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764406.596993:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764406.596995:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311168:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764407.172737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764407.620632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764408.196639:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764408.644618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764409.220682:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764409.668621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764410.244659:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764410.692593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764411.268687:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764411.716619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764411.716622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764411.716628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764411.716631:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764411.716640:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3456 00000100:00080000:1.0:1787764411.716650:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764411.716658:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764411.716693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274311296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764411.716728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764411.716731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764411.716732:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764411.716735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3456 00000100:00080000:1.0:1787764411.716738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764411.716766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764411.716773:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311296:192.168.203.123@tcp:250:kworker.0 00000008:00100000:1.0:1787764411.716775:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764411.716788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274311424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764411.716834:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764411.716871:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764411/real 1787764411] req@ffff9ac4478421c0 x1874603274311296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764411.716894:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274311296/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764411.716910:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764411.716912:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764411.716915:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311296:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764411.716931:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311424:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764411.716947:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764411.716954:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764411/real 1787764411] req@ffff9ac4478430c0 x1874603274311424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764411.716962:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274311424/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764411.716966:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764411.716968:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764411.716970:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311424:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764412.292695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764412.740646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764413.316695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764413.764654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764414.340724:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764414.788682:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764415.364681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764415.812621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764416.388711:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764416.836634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764416.836643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764416.836650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764416.836654:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764416.836664:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3461 00000100:00080000:0.0:1787764416.836674:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764416.836682:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764416.836711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274311552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764416.836742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764416.836761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764416.836762:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764416.836766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3461 00000100:00080000:0.0:1787764416.836770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787764416.836773:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311552:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787764416.836776:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764416.836787:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764416.836799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274311680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787764416.836828:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764416.836861:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764416/real 1787764416] req@ffff9ac451f84b40 x1874603274311552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764416.836875:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274311552/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764416.836882:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764416.836885:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764416.836887:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311552:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764416.836906:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311680:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764416.836927:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764416.836940:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764416/real 1787764416] req@ffff9ac451f87c00 x1874603274311680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764416.836949:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274311680/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764416.836959:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764416.836961:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764416.836964:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311680:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764417.412666:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764417.860627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764418.436739:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764418.884637:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764419.460638:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764419.908617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764420.484674:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764420.932621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764421.508652:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764421.956615:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764421.956653:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764421.956660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764421.956664:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764421.956674:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3466 00000100:00080000:2.0:1787764421.956684:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764421.956692:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764421.956723:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bcf00 x1874603274311808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764421.956760:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764421.956764:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764421.956766:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764421.956771:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3466 00000100:00080000:2.0:1787764421.956775:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764421.956780:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764421.956789:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311808:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787764421.956790:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764421.956808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc780 x1874603274311936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787764421.956849:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764421.956890:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764421/real 1787764421] req@ffff9ac4523bcf00 x1874603274311808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764421.956904:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bcf00 x1874603274311808/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764421.956911:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764421.956914:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764421.956917:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274311808:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764421.956934:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311936:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764421.956949:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764421.956956:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764421/real 1787764421] req@ffff9ac4523bc780 x1874603274311936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764421.956964:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc780 x1874603274311936/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764421.956979:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764421.956982:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764421.956985:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274311936:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764422.532662:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764422.980635:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764423.556651:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764424.004601:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764424.580664:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764425.028620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764425.604670:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764426.052618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764426.628667:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764427.076642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764427.076645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764427.076653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764427.076658:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764427.076667:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3471 00000100:00080000:0.0:1787764427.076677:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764427.076684:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764427.076731:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274312064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764427.076755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764427.076758:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764427.076759:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764427.076761:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3471 00000100:00080000:0.0:1787764427.076764:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764427.076766:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764427.076774:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764427.076783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274312192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764427.076840:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312064:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787764427.076899:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764427.076941:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764427/real 1787764427] req@ffff9ac451f87c00 x1874603274312064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764427.076962:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274312064/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764427.076969:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764427.076972:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764427.076975:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312064:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764427.076980:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312192:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764427.076994:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764427.077003:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764427/real 1787764427] req@ffff9ac451f84b40 x1874603274312192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764427.077015:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274312192/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764427.077036:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764427.077040:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764427.077043:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312192:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764427.652699:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764428.100641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764428.676696:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764429.124629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764429.700673:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764430.148621:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764430.724633:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764431.172625:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764431.748683:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764432.196642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764432.196645:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764432.196653:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764432.196661:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764432.196671:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3476 00000100:00080000:1.0:1787764432.196680:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764432.196687:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764432.196717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274312320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764432.196744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764432.196748:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764432.196749:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764432.196754:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3476 00000100:00080000:1.0:1787764432.196757:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764432.196762:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764432.196772:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764432.196784:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764432.196785:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274312448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764432.196845:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764432.196878:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764432/real 1787764432] req@ffff9ac4478403c0 x1874603274312320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764432.196890:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274312320/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764432.196897:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764432.196900:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764432.196903:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312320:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764432.196919:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312448:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764432.196933:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764432.196940:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764432/real 1787764432] req@ffff9ac447840780 x1874603274312448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764432.196947:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274312448/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764432.196952:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764432.196954:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764432.196956:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312448:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764432.772673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764433.220613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764433.796695:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764434.244632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764434.820691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764435.268645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764435.844715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764436.292671:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764436.868649:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764437.316608:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764437.316613:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764437.316624:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764437.316628:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764437.316638:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3482 00000100:00080000:1.0:1787764437.316649:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764437.316656:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764437.316695:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274312576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764437.316729:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764437.316731:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764437.316733:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764437.316736:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3482 00000100:00080000:1.0:1787764437.316738:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764437.316740:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764437.316753:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764437.316762:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274312704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764437.316788:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312576:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764437.316866:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764437.316933:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764437/real 1787764437] req@ffff9ac447841680 x1874603274312576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764437.316952:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274312576/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764437.316964:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764437.316968:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764437.316973:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312576:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764437.316981:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312704:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764437.317003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764437.317016:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764437/real 1787764437] req@ffff9ac447840f00 x1874603274312704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764437.317029:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274312704/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764437.317038:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764437.317041:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764437.317043:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312704:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764437.892727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764438.340646:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764438.916673:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764439.364622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764439.940715:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764440.388627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764440.964979:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764441.412617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764441.988697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764442.436648:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764442.436661:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764442.436686:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764442.436690:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764442.436702:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3487 00000100:00080000:1.0:1787764442.436714:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764442.436721:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764442.436759:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274312832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764442.436792:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764442.436796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764442.436797:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764442.436801:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3487 00000100:00080000:1.0:1787764442.436804:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764442.436806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764442.436814:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764442.436827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274312960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764442.436849:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312832:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764442.436935:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764442.436987:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764442/real 1787764442] req@ffff9ac447840000 x1874603274312832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764442.437008:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274312832/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764442.437029:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764442.437033:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764442.437036:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274312832:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764442.437064:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312960:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764442.437083:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764442.437091:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764442/real 1787764442] req@ffff9ac447842580 x1874603274312960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764442.437099:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274312960/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764442.437104:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764442.437113:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764442.437117:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274312960:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764443.013412:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764443.460663:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764444.036691:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764444.484683:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764445.061067:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764445.508672:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764446.085353:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764446.532673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764447.108794:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764447.556631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764447.556672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764447.556678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764447.556682:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764447.556690:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3492 00000100:00080000:0.0:1787764447.556700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764447.556706:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764447.556736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274313088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764447.556792:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313088:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787764447.556848:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00080000:0.0:1787764447.556874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764447.556893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764447.556895:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764447.556900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3492 00000100:00080000:0.0:1787764447.556904:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764447.556908:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764447.556916:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764447.556928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274313216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764447.556979:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764447/real 1787764447] req@ffff9ac451f870c0 x1874603274313088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764447.556994:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274313088/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764447.557015:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764447.557019:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764447.557024:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313088:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764447.557044:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313216:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764447.557065:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764447.557075:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764447/real 1787764447] req@ffff9ac451f84000 x1874603274313216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764447.557098:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274313216/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764447.557107:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764447.557110:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764447.557113:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313216:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764448.132697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764448.580631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764449.158067:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764449.604645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764450.180669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764450.628633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764451.204708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764451.652641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764452.228730:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764452.676616:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764452.676798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764452.676804:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764452.676808:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764452.676817:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3497 00000100:00080000:2.0:1787764452.676827:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764452.676832:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764452.676860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bd680 x1874603274313344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764452.676882:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764452.676885:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764452.676891:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764452.676895:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3497 00000100:00080000:2.0:1787764452.676898:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764452.676902:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764452.676921:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313344:192.168.203.123@tcp:250:kworker.0 00000008:00100000:2.0:1787764452.676924:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764452.676936:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bc000 x1874603274313472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787764452.676983:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764452.677181:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764452/real 1787764452] req@ffff9ac4523bd680 x1874603274313344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764452.677199:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bd680 x1874603274313344/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764452.677222:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764452.677225:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764452.677228:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bd680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313344:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764452.677249:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313472:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764452.677272:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764452.677284:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764452/real 1787764452] req@ffff9ac4523bc000 x1874603274313472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764452.677293:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bc000 x1874603274313472/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764452.677298:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764452.677300:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764452.677302:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313472:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764453.253178:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764453.700610:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764454.276690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764454.725057:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764455.300678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764455.748760:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764456.324768:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764456.772673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764457.348731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764457.796674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764457.796675:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764457.796686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764457.796692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764457.796702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3502 00000100:00080000:1.0:1787764457.796716:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764457.796724:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764457.796766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274313600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764457.796821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764457.796825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764457.796826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764457.796831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3502 00000100:00080000:1.0:1787764457.796835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764457.796839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764457.796851:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764457.796864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274313728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764457.796905:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313600:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764457.796972:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764457.797011:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764457/real 1787764457] req@ffff9ac447843840 x1874603274313600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764457.797044:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274313600/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764457.797055:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764457.797059:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764457.797063:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313600:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764457.797071:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313728:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764457.797091:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764457.797102:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764457/real 1787764457] req@ffff9ac447841a40 x1874603274313728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764457.797114:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274313728/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764457.797133:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764457.797137:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764457.797140:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313728:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764458.372737:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764458.820599:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764459.396668:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764459.844622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764460.420656:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764460.868593:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764461.444675:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764461.892639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764462.468637:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764462.916622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764462.916629:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764462.916635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764462.916638:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764462.916646:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3507 00000100:00080000:0.0:1787764462.916654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764462.916660:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764462.916685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274313856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764462.916745:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313856:192.168.203.123@tcp:250:kworker.0 00000100:00080000:0.0:1787764462.916847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764462.916852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764462.916854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764462.916859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3507 00000100:00080000:0.0:1787764462.916863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764462.916867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764462.916874:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764462.916885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274313984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764462.916930:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313984:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764463.492702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764463.940641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764464.516690:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764464.964650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764465.540743:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787764465.988813:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000100:00100000:3.0:1787764465.988813:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787764465.988858:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787764465.988881:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:2.0:1787764465.988922:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:1.0:1787764465.989008:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764462/real 1787764465] req@ffff9ac451f84780 x1874603274313984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764465.989030:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274313984/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764465.989042:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764465.989047:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764465.989051:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274313984:192.168.203.123@tcp:38:kworker.0 00000100:00100000:1.0:1787764465.989057:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764462/real 1787764465] req@ffff9ac451f861c0 x1874603274313856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764465.989091:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274313856/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764465.989099:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764465.989103:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764465.989106:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274313856:192.168.203.123@tcp:250:kworker.0 00010000:00010000:3.0:1787764466.564698:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764467.012628:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764467.588732:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764468.036622:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764468.036653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764468.036661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764468.036665:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764468.036676:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3512 00000100:00080000:1.0:1787764468.036686:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764468.036693:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764468.036731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274314112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764468.036764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764468.036768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764468.036769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764468.036774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3512 00000100:00080000:1.0:1787764468.036778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764468.036782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764468.036794:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764468.036808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274314240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764468.036822:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314112:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764468.036903:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764468.036989:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764468/real 1787764468] req@ffff9ac4478412c0 x1874603274314112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764468.037009:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274314112/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764468.037020:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764468.037024:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764468.037028:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314112:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764468.037057:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274314240:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764468.037080:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764468.037093:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764468/real 1787764468] req@ffff9ac447841e00 x1874603274314240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764468.037105:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274314240/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764468.037112:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764468.037115:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764468.037118:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274314240:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764468.612677:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764469.060652:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764469.636642:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764470.084644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764470.660701:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764471.108620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764471.684695:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764472.133241:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764472.708681:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764473.156620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764473.156620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764473.156629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764473.156633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764473.156642:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3517 00000100:00080000:1.0:1787764473.156652:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764473.156658:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764473.156689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843480 x1874603274314368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764473.156714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764473.156718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764473.156720:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764473.156724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3517 00000100:00080000:1.0:1787764473.156727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764473.156731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764473.156739:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764473.156752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841e00 x1874603274314496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764473.156789:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314368:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764473.156864:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764473.156906:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764473/real 1787764473] req@ffff9ac447843480 x1874603274314368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764473.156932:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843480 x1874603274314368/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764473.156962:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764473.156967:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764473.156971:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314368:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764473.156979:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274314496:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764473.157003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764473.157016:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764473/real 1787764473] req@ffff9ac447841e00 x1874603274314496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764473.157029:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841e00 x1874603274314496/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764473.157037:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764473.157040:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764473.157042:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274314496:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764473.732624:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764474.180606:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764474.756727:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764475.204620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764475.780666:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764476.228645:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764476.804724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764477.252641:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764477.828728:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764478.276632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764478.276655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764478.276662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764478.276668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764478.276678:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3523 00000100:00080000:0.0:1787764478.276689:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764478.276696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764478.276733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274314624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764478.276761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764478.276783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764478.276785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764478.276791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3523 00000100:00080000:0.0:1787764478.276796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764478.276801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764478.276809:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314624:192.168.203.123@tcp:250:kworker.0 00000008:00100000:0.0:1787764478.276813:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764478.276829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274314752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787764478.276893:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764478.276931:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764478/real 1787764478] req@ffff9ac451f85680 x1874603274314624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764478.276944:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274314624/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764478.276956:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764478.276965:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764478.276968:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314624:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764478.276991:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274314752:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764478.277007:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764478.277015:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764478/real 1787764478] req@ffff9ac451f843c0 x1874603274314752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764478.277022:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274314752/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764478.277026:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764478.277029:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764478.277030:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274314752:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764478.852697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764479.300617:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764479.876691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764480.324624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764480.900671:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764481.348632:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764481.924685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764482.372620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764482.948692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764483.396620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787764483.396620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764483.396629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764483.396634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764483.396645:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3528 00000100:00080000:0.0:1787764483.396655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764483.396662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764483.396697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f843c0 x1874603274314880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764483.396724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764483.396728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764483.396730:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764483.396734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3528 00000100:00080000:0.0:1787764483.396738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764483.396743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764483.396752:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764483.396773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f85680 x1874603274315008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764483.396788:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314880:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787764483.396883:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764483.396945:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764483/real 1787764483] req@ffff9ac451f843c0 x1874603274314880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764483.396970:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f843c0 x1874603274314880/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764483.396983:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764483.396999:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764483.397004:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f843c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274314880:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764483.397047:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315008:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764483.397072:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764483.397085:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764483/real 1787764483] req@ffff9ac451f85680 x1874603274315008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764483.397099:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f85680 x1874603274315008/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764483.397107:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764483.397111:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764483.397114:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f85680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315008:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764483.972677:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764484.420624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764484.996665:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764485.444620:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764486.020687:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764486.468624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764487.044728:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764487.492650:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764488.068763:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764488.516647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764488.516656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764488.516664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764488.516669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764488.516680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3533 00000100:00080000:0.0:1787764488.516694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764488.516703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764488.516751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f861c0 x1874603274315136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764488.516783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764488.516786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764488.516788:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764488.516791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3533 00000100:00080000:0.0:1787764488.516794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764488.516797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764488.516821:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764488.516832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84780 x1874603274315264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764488.516861:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315136:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787764488.516942:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764488.516984:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764488/real 1787764488] req@ffff9ac451f861c0 x1874603274315136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764488.516997:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f861c0 x1874603274315136/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764488.517005:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764488.517007:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764488.517010:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f861c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315136:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764488.517015:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315264:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764488.517031:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764488.517038:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764488/real 1787764488] req@ffff9ac451f84780 x1874603274315264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764488.517063:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84780 x1874603274315264/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764488.517068:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764488.517070:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764488.517072:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315264:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764489.092772:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764489.540668:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764490.116731:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764490.564639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764491.140686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764491.588662:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764492.165967:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764492.612909:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764493.188800:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764493.636656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764493.636665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764493.636670:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764493.636679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3538 00000100:00080000:1.0:1787764493.636689:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764493.636697:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764493.636745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478412c0 x1874603274315392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764493.636783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764493.636787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764493.636789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764493.636794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3538 00000100:00080000:1.0:1787764493.636798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764493.636803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764493.636814:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764493.636820:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764493.636830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841a40 x1874603274315520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764493.636889:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764493.636924:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764493/real 1787764493] req@ffff9ac4478412c0 x1874603274315392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764493.636940:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478412c0 x1874603274315392/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764493.636955:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764493.636959:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764493.636963:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478412c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315392:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764493.636983:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315520:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764493.637003:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764493.637013:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764493/real 1787764493] req@ffff9ac447841a40 x1874603274315520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764493.637024:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841a40 x1874603274315520/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764493.637033:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764493.637036:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764493.637039:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315520:192.168.203.123@tcp:38:kworker.0 00000100:00100000:3.0:1787764493.637271:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764494.212719:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764494.660746:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764495.236678:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764495.685272:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764496.261471:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764496.708665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764497.285585:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764497.732626:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764498.309396:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764498.756804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764498.756813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764498.756817:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764498.756830:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3543 00000100:00080000:0.0:1787764498.756842:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764498.756849:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764498.756882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84000 x1874603274315648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764498.756917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764498.756939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764498.756941:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764498.756946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3543 00000100:00080000:0.0:1787764498.756950:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764498.756955:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764498.756964:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764498.756978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f870c0 x1874603274315776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764498.757065:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315648:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787764498.757121:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764498.757164:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764498/real 1787764498] req@ffff9ac451f84000 x1874603274315648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764498.757181:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84000 x1874603274315648/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764498.757190:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764498.757194:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764498.757198:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315648:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764498.757205:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315776:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764498.757226:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764498.757240:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764498/real 1787764498] req@ffff9ac451f870c0 x1874603274315776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764498.757261:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f870c0 x1874603274315776/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764498.757271:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787764498.757271:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764498.757276:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764498.757283:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f870c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274315776:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764499.332703:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764499.780612:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764500.356699:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764500.804654:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764501.380856:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764501.828684:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764502.405610:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764502.852634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764503.428672:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764503.876642:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764503.876749:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764503.876756:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764503.876773:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764503.876782:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3548 00000100:00080000:1.0:1787764503.876791:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764503.876798:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764503.876829:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447843840 x1874603274315904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764503.876856:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764503.876859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764503.876861:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764503.876866:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3548 00000100:00080000:1.0:1787764503.876869:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764503.876873:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764503.876883:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764503.876894:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842580 x1874603274316032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764503.877309:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315904:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764503.877357:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764503.877390:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764503/real 1787764503] req@ffff9ac447843840 x1874603274315904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764503.877401:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447843840 x1874603274315904/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764503.877408:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764503.877410:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764503.877413:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447843840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274315904:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764503.877417:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316032:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764503.877430:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764503.877437:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764503/real 1787764503] req@ffff9ac447842580 x1874603274316032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764503.877444:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842580 x1874603274316032/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764503.877455:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764503.877457:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764503.877459:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316032:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764504.452702:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764504.900653:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764505.476711:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764505.924630:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764506.500727:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764506.948657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764507.524720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764507.972660:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764508.548720:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787764508.996674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764508.996684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764508.996689:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764508.996698:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3553 00000100:00080000:0.0:1787764508.996708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764508.996715:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764508.996749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274316160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764508.996780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764508.996784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764508.996786:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764508.996790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3553 00000100:00080000:0.0:1787764508.996793:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764508.996797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764508.996806:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764508.996816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274316288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787764508.997272:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316160:192.168.203.123@tcp:250:kworker.0 00000800:00000100:1.0:1787764508.997337:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764508.997396:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764508/real 1787764508] req@ffff9ac451f84b40 x1874603274316160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764508.997417:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274316160/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764508.997427:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764508.997431:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764508.997447:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316160:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764508.997455:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316288:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764508.997479:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764508.997492:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764508/real 1787764508] req@ffff9ac451f87c00 x1874603274316288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764508.997505:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274316288/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764508.997514:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764508.997517:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764508.997521:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316288:192.168.203.123@tcp:38:kworker.0 00000100:00100000:3.0:1787764508.997737:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764509.572731:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764510.020688:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764510.596642:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764511.044765:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764511.621697:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764512.068634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764512.644685:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764513.092644:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764513.668732:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764514.116673:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787764514.116676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764514.116686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787764514.116707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764514.116717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3558 00000100:00080000:0.0:1787764514.116729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764514.116738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:0.0:1787764514.116780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f87c00 x1874603274316416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787764514.116808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787764514.116811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787764514.116812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787764514.116816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3558 00000100:00080000:0.0:1787764514.116819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787764514.116823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:0.0:1787764514.116834:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764514.116847:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764514.116851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac451f84b40 x1874603274316544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787764514.116940:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764514.116989:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764514/real 1787764514] req@ffff9ac451f87c00 x1874603274316416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764514.117008:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f87c00 x1874603274316416/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764514.117019:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764514.117023:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764514.117027:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f87c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316416:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764514.117052:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316544:192.168.203.123@tcp:38:kworker.0 00000800:00000100:1.0:1787764514.117074:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:1.0:1787764514.117096:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764514/real 1787764514] req@ffff9ac451f84b40 x1874603274316544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787764514.117108:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac451f84b40 x1874603274316544/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787764514.117117:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787764514.117120:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:1.0:1787764514.117124:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac451f84b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316544:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764514.692691:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764515.140670:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764515.716720:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764516.164627:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764516.740736:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764517.188631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764517.764686:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764518.212666:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764518.788645:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764519.236607:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764519.236616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764519.236623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764519.236627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764519.236636:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3563 00000100:00080000:1.0:1787764519.236647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764519.236654:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764519.236687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840000 x1874603274316672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764519.236719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764519.236726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764519.236728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764519.236732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3563 00000100:00080000:1.0:1787764519.236736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764519.236742:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764519.236751:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764519.236758:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764519.236766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840f00 x1874603274316800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764519.236816:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764519.236851:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764519/real 1787764519] req@ffff9ac447840000 x1874603274316672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764519.236868:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840000 x1874603274316672/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764519.236878:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764519.236882:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764519.236891:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316672:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764519.236911:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316800:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764519.236932:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764519.236945:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764519/real 1787764519] req@ffff9ac447840f00 x1874603274316800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764519.236957:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840f00 x1874603274316800/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764519.236966:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764519.236969:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764519.236972:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274316800:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764519.812685:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764520.260648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764520.836661:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764521.284633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764521.860669:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764522.308691:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764522.884705:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764523.332631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764523.908702:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764524.356618:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787764524.356640:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764524.356645:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787764524.356654:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764524.356661:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3569 00000100:00080000:2.0:1787764524.356670:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764524.356675:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:2.0:1787764524.356701:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523be940 x1874603274316928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787764524.356719:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787764524.356721:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787764524.356722:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787764524.356725:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3569 00000100:00080000:2.0:1787764524.356727:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787764524.356729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:2.0:1787764524.356736:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787764524.356743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4523bda40 x1874603274317056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764524.356757:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316928:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764524.356805:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764524.356833:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764524/real 1787764524] req@ffff9ac4523be940 x1874603274316928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764524.356849:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523be940 x1874603274316928/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764524.356859:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764524.356862:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764524.356866:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523be940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274316928:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764524.356888:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317056:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764524.356904:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764524.356919:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764524/real 1787764524] req@ffff9ac4523bda40 x1874603274317056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764524.356931:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4523bda40 x1874603274317056/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764524.356939:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764524.356942:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764524.356946:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4523bda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317056:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764524.932678:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764525.380647:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764525.956675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764526.404665:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764526.980698:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764527.428633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764528.004704:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764528.452639:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764529.028694:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764529.476680:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764529.478581:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764529.478590:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764529.478594:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764529.478605:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3574 00000100:00080000:1.0:1787764529.478618:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764529.478627:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764529.478673:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447841680 x1874603274317184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764529.478705:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764529.478709:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764529.478711:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764529.478716:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3574 00000100:00080000:1.0:1787764529.478719:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764529.478723:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764529.478734:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764529.478750:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840780 x1874603274317312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764529.478796:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764529.478913:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764530.052675:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764530.500658:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764531.076715:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764531.524664:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764532.100692:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764532.548623:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000400:00000100:1.0:1787764532.548801:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.123:988 00000400:00000100:1.0:1787764532.548832:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.123@tcp at host 192.168.203.123 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787764532.548847:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000800:00000100:1.0:1787764532.548890:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.123@tcp 00000100:00100000:0.0:1787764532.548970:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764529/real 1787764532] req@ffff9ac447841680 x1874603274317184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764532.548993:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447841680 x1874603274317184/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764532.549008:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764532.549013:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764532.549017:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447841680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317184:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764532.549025:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764529/real 1787764532] req@ffff9ac447840780 x1874603274317312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764532.549037:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840780 x1874603274317312/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764532.549044:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764532.549061:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764532.549064:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317312:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764533.124689:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764533.572648:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764534.149061:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787764534.596756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764534.596766:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764534.596784:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764534.596794:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3579 00000100:00080000:1.0:1787764534.596804:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764534.596811:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764534.596843:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478403c0 x1874603274317440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764534.596994:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764534.596998:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764534.597000:0:38:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764534.597005:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3579 00000100:00080000:1.0:1787764534.597010:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764534.597015:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764534.597025:0:38:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764534.597038:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478430c0 x1874603274317568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764534.597132:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317440:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764534.597194:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764534.597236:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764534/real 1787764534] req@ffff9ac4478403c0 x1874603274317440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764534.597255:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478403c0 x1874603274317440/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764534.597266:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764534.597270:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764534.597275:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478403c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317440:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764534.597282:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317568:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764534.597304:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764534.597319:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764534/real 1787764534] req@ffff9ac4478430c0 x1874603274317568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764534.597333:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478430c0 x1874603274317568/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764534.597344:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764534.597348:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:3.0:1787764534.597351:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787764534.598747:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478430c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317568:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764535.172706:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764535.620624:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764536.197248:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764536.645657:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764537.220815:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764537.669440:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764538.245056:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764538.692638:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764539.268681:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764539.716634:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764539.716637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764539.716645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764539.716649:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764539.716658:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3584 00000100:00080000:1.0:1787764539.716669:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764539.716677:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764539.716713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac4478421c0 x1874603274317696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764539.716758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764539.716761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764539.716763:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764539.716766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3584 00000100:00080000:1.0:1787764539.716769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764539.716772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764539.716780:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787764539.716791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842d00 x1874603274317824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787764539.716808:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317696:192.168.203.123@tcp:250:kworker.0 00000800:00000100:0.0:1787764539.716867:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764539.716914:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764539/real 1787764539] req@ffff9ac4478421c0 x1874603274317696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764539.716931:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac4478421c0 x1874603274317696/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764539.716940:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764539.716944:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764539.716948:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac4478421c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317696:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764539.716968:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317824:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764539.716988:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764539.716999:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764539/real 1787764539] req@ffff9ac447842d00 x1874603274317824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764539.717011:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842d00 x1874603274317824/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764539.717019:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764539.717022:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764539.717026:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274317824:192.168.203.123@tcp:38:kworker.0 00010000:00010000:1.0:1787764540.292738:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764540.740633:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764541.316708:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764541.764629:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764542.340700:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787764542.788862:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787764543.364746:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764543.812669:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764544.388724:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764544.836631:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787764544.836713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 94b87323-e84d-4c2a-846c-789bad072fad->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764544.836718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787764544.836721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764544.836729:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.123@tcp: connect to NID 192.168.203.123@tcp last attempt 3589 00000100:00080000:1.0:1787764544.836737:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.123@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764544.836742:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.123@tcp: import ffff9ac448a0d800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000100:00100000:1.0:1787764544.836770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447840b40 x1874603274317952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787764544.836795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 522fbd36-e7cb-44ce-913c-7337604532e0->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787764544.836799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787764544.836801:0:36:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787764544.836805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: connect to NID 192.168.203.123@tcp last attempt 3589 00000100:00080000:1.0:1787764544.836824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787764544.836828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9ac4523c1000: import ffff9ac4454bf800 using connection 192.168.203.123@tcp/192.168.203.123@tcp 00000008:00100000:1.0:1787764544.836837:0:36:0:(osc_request.c:4034:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683255 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787764544.836839:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:1.0:1787764544.836849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9ac447842940 x1874603274318080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787764544.836890:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764544.836921:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764544/real 1787764544] req@ffff9ac447840b40 x1874603274317952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764544.836942:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447840b40 x1874603274317952/t0(0) o250->MGC192.168.203.123@tcp@192.168.203.123@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764544.836949:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac448a0d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764544.836951:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764544.836954:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447840b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:94b87323-e84d-4c2a-846c-789bad072fad:2344:1874603274317952:192.168.203.123@tcp:250:kworker.0 00000100:00100000:0.0:1787764544.836970:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274318080:192.168.203.123@tcp:38:kworker.0 00000800:00000100:0.0:1787764544.836984:0:2344:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.123@tcp 00000100:00100000:0.0:1787764544.836992:0:2344:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764544/real 1787764544] req@ffff9ac447842940 x1874603274318080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787764544.836999:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9ac447842940 x1874603274318080/t0(0) o38->lustre-MDT0000-mdc-ffff9ac4523c1000@192.168.203.123@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787764544.837004:0:2344:0:(import.c:65:import_set_state_nolock()) ffff9ac4454bf800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787764544.837006:0:2344:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.123@tcp failed (-110) 00000100:00100000:0.0:1787764544.837008:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9ac447842940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:522fbd36-e7cb-44ce-913c-7337604532e0:2344:1874603274318080:192.168.203.123@tcp:38:kworker.0 00010000:00010000:3.0:1787764545.412784:0:5438:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787764545.860659:0:9835:0:(client.c:2715:ptlrpc_set_wait()) set ffff9ac4463689c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787764546.437229:0:5439:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 26394 lines, 26394 kept, 0 dropped, 0 bad.